From 283fb2aac0db12ac34641b53b4d06e981417d365 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 02:09:08 +0000 Subject: [PATCH 01/12] fix(engine): fence cancels on the request's incarnation (FIX-1654) The abort route fenced its conditional write on the checked record's createdAt, which is wrong both ways: a same-owner hand-off rewrites createdAt (the owner's cancel 404s and the run keeps going), and two requests under one id can share a millisecond (another tenant's cancel lands). setFieldsIfStatus now takes expectedIncarnation, compared with the stored record's resolved incarnation inside each store's atomic step; resolveRequestIncarnation is exported for out-of-tree stores. The route's in-process fire is fenced the same way: registered abort controllers carry their request's incarnation, runAction registers with it, and the host tags its queued pre-registration once the record is claimed. Callers with no record to check (CLI, cascading child abort) still act on the id alone. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- .changeset/abort-fence-incarnation-stores.md | 6 + .changeset/abort-fence-incarnation.md | 9 + docs/architecture/state-and-scopes.md | 2 +- packages/engine/README.md | 9 +- .../engine/src/execution/abort-registry.ts | 65 ++++- packages/engine/src/execution/index.ts | 3 +- packages/engine/src/execution/runAction.ts | 21 +- packages/engine/src/index.ts | 1 + packages/engine/src/routes/abort-routes.ts | 22 +- .../src/stores/filesystem/request-store.ts | 9 +- packages/engine/src/stores/index.ts | 3 +- .../engine/src/stores/memory/request-store.ts | 9 +- .../testing/request-store-conformance.ts | 140 +++++++-- packages/engine/src/stores/types.ts | 24 +- .../host/createInboundTransportHost.ts | 25 +- packages/engine/test/abort.test.ts | 267 +++++++++++++++++- .../abort-fenced-on-incarnation.test.ts | 219 ++++++++++++++ packages/store-postgres/src/request-store.ts | 15 +- packages/store-sqlite/src/request-store.ts | 9 +- 19 files changed, 774 insertions(+), 84 deletions(-) create mode 100644 .changeset/abort-fence-incarnation-stores.md create mode 100644 .changeset/abort-fence-incarnation.md create mode 100644 packages/integration-tests/src/scenarios/abort-fenced-on-incarnation.test.ts diff --git a/.changeset/abort-fence-incarnation-stores.md b/.changeset/abort-fence-incarnation-stores.md new file mode 100644 index 0000000000..0a4201241c --- /dev/null +++ b/.changeset/abort-fence-incarnation-stores.md @@ -0,0 +1,6 @@ +--- +"@flow-state-dev/store-sqlite": patch +"@flow-state-dev/store-postgres": patch +--- + +`setFieldsIfStatus` fences a cancel on the request's incarnation instead of its `createdAt`, so a cancel reaches the owner's request after their own retry re-stamped it, and never a later request that reused the id (FIX-1654). No schema change; records written before incarnations existed are matched on their `legacy_` value. diff --git a/.changeset/abort-fence-incarnation.md b/.changeset/abort-fence-incarnation.md new file mode 100644 index 0000000000..c9ce2d429c --- /dev/null +++ b/.changeset/abort-fence-incarnation.md @@ -0,0 +1,9 @@ +--- +"@flow-state-dev/engine": minor +--- + +A cancel now lands on the request its caller checked, identified by incarnation rather than `createdAt` (FIX-1654). Before, a cancel issued while the owner's own retry re-stamped the request answered 404 and never stopped the run, and a request created in the same millisecond under a reused id could receive another caller's cancel. + +**Store authors:** `RequestStore.setFieldsIfStatus`'s fifth parameter is now `expectedIncarnation?: string`, replacing `expectedCreatedAt?: number`. Compare it with the stored record's `resolveRequestIncarnation(record)` (newly exported) inside the same atomic step as the status check, and report a mismatch as an absent record. See the engine README, "Abort intent on `RequestStore`". + +The in-process abort registry takes the incarnation too: `registerAbortController(id, incarnation?)`, `abortRequest(id, expectedIncarnation?)` and `hasActiveAbortController(id, expectedIncarnation?)`, plus a new `tagAbortController(id, controller, incarnation)`. Callers that pass no incarnation behave as before. diff --git a/docs/architecture/state-and-scopes.md b/docs/architecture/state-and-scopes.md index 5f09d9595e..30cbae2928 100644 --- a/docs/architecture/state-and-scopes.md +++ b/docs/architecture/state-and-scopes.md @@ -524,7 +524,7 @@ Records written before this rule attach, resume and are retried by their owner e The binding above is to the record, and retention deletes records. Once one is gone its id is free: the next request under it, from anyone, is a new request that no check refuses. State keyed on the id alone that outlives the record, such as a local `scope: "run"` bash workspace directory, would pass to that new request. So every request record carries an **incarnation** (`RequestRecord.incarnation`), a random token minted once by `createInitialRequestRecord` and never rewritten: `claimRequestRecord`'s same-owner hand-off keeps the stored value and returns the record as written, and every context (a retry, a queued run adopting its host's stub, a resume) reads it from the record it claimed or adopted. It reaches blocks as `ctx.request.incarnation`. A record written before the field existed answers to a value derived from its `createdAt` under a `legacy_` prefix no minted token uses (`resolveRequestIncarnation`, BP-030). `isSameRequest` compares incarnations, so two records under one id created in the same millisecond are still two requests. -The workspace package's request scope identity is `[tenant, requestId, incarnation]` (`packages/workspace/src/scope-identity.ts`), and the bash run workspace key and directory derive from it. Anything newly keyed on a request id that can outlive the request keys on that identity, not on the id. Known exceptions to settle later: FIX-1021's abort fence (`setFieldsIfStatus(..., expectedCreatedAt)`) still fences on `createdAt`. +The workspace package's request scope identity is `[tenant, requestId, incarnation]` (`packages/workspace/src/scope-identity.ts`), and the bash run workspace key and directory derive from it. Anything newly keyed on a request id that can outlive the request keys on that identity, not on the id. The cancel route's conditional write is fenced the same way: `setFieldsIfStatus` takes the incarnation the route's owner check read, and every store compares it with the stored record's, so a cancel never lands on a later request under the id and never misses its own request after a same-owner hand-off. The route's in-process fire checks the same incarnation: each registered abort controller carries the incarnation of the request it belongs to, and a fenced fire acts only on a match. ## Cross-Flow State: Shared vs Isolated diff --git a/packages/engine/README.md b/packages/engine/README.md index 200817ea1a..461f62efcf 100644 --- a/packages/engine/README.md +++ b/packages/engine/README.md @@ -569,7 +569,8 @@ setFieldsIfStatus( id: string, fields: ConditionalRequestFields, allowedStatuses: readonly RequestStatus[], - updatedAt: number + updatedAt: number, + expectedIncarnation?: string ): Promise; ``` @@ -579,6 +580,12 @@ Both named types are exported from `@flow-state-dev/engine`. `ConditionalRequest **`setFieldsIfStatus`** applies `fields` only while the record's status is one of `allowedStatuses`, evaluating the predicate and the write as one atomic step. Three outcomes: `{ applied: true, status }` when the predicate held; `{ applied: false, status }` when a record exists outside the predicate; `{ applied: false, status: undefined }` when no record exists. +**`expectedIncarnation` fences the write to one request.** A request id can be reused once retention deletes its record, so the record at `id` when you write may not be the one the caller checked. When `expectedIncarnation` is given, compare it with the stored record's incarnation, resolved with `resolveRequestIncarnation(record)` (exported from `@flow-state-dev/engine`), and when they differ report the record as absent, `{ applied: false, status: undefined }`, and write nothing. Do the comparison inside the same atomic step as the status check. + +Resolve the stored side with the helper rather than reading `record.incarnation` directly. Records written before incarnations existed have none, and the helper gives them a stable value derived from `createdAt`. If your store has to state the rule in its own query language, the derived form is `legacy_` followed by `createdAt` in decimal, used when `incarnation` is absent or `null`. + +Don't compare `createdAt` instead. Two requests under one id can be created in the same millisecond, and a retry by the same owner rewrites `createdAt` on a request that is still running. + Do not implement it as a version CAS. Terminal transitions persist `version` **unchanged**, so a version-checked write still validates after a terminal commit and resurrects a dead record. The predicate has to read status. **`setFieldsIfStatus` is the only member that may write `abortRequested`.** `set` must ignore the field in both directions: a full record handed to `set` cannot set the flag and cannot clear a stored one. The compiler will not enforce this, since the field is still on `RequestRecord` for reading. An adapter that honours it on `set` lets a full-record write built from a stale snapshot erase a cancellation. diff --git a/packages/engine/src/execution/abort-registry.ts b/packages/engine/src/execution/abort-registry.ts index 409446411e..020b18ab5b 100644 --- a/packages/engine/src/execution/abort-registry.ts +++ b/packages/engine/src/execution/abort-registry.ts @@ -10,30 +10,73 @@ * process, and `runAction`'s heartbeat poll when the intent was recorded * somewhere else — so a cross-process abort is indistinguishable downstream * from a local one. + * + * A request id names one request only while its record exists, so the id alone + * cannot tell a checked request from a later one that reused it. A controller + * therefore carries the incarnation of the request it belongs to, and a caller + * that checked a record passes that record's incarnation to fire or probe only + * that request's controller. Callers with no record to check (a CLI's own + * cancel, a parent cascading to its child) pass none and act on the id alone. */ -const controllers = new Map(); +interface Registered { + controller: AbortController; + /** The incarnation of the request this controller belongs to, once known. */ + incarnation?: string; +} + +const controllers = new Map(); + +/** True when `entry` may be acted on by a caller expecting `expected`. */ +function matches(entry: Registered, expected: string | undefined): boolean { + return expected === undefined || entry.incarnation === expected; +} /** * Register an AbortController for a request. Returns the controller * so the caller can use its signal. + * + * Pass the request's incarnation when it is known, so a fenced + * `abortRequest` can tell this request from a later one under the same id. + * An untagged controller is never fired by a fenced abort; tag it later with + * `tagAbortController` once the incarnation is known. */ -export function registerAbortController(requestId: string): AbortController { +export function registerAbortController(requestId: string, incarnation?: string): AbortController { const controller = new AbortController(); - controllers.set(requestId, controller); + controllers.set(requestId, incarnation === undefined ? { controller } : { controller, incarnation }); return controller; } +/** + * Record the incarnation of an already-registered controller. Has no effect + * when `controller` is no longer the one registered under `requestId`, so a + * late tag cannot label a later request's controller. + */ +export function tagAbortController( + requestId: string, + controller: AbortController, + incarnation: string +): void { + const entry = controllers.get(requestId); + if (entry !== undefined && entry.controller === controller) { + entry.incarnation = incarnation; + } +} + /** * Signal abort for a request. Returns true if the request was found * and aborted, false if the request was not in the registry. + * + * With `expectedIncarnation`, fires only when the registered controller + * belongs to that incarnation; a controller of another request under the same + * id, or one not yet tagged, is left alone and the call returns false. */ -export function abortRequest(requestId: string): boolean { - const controller = controllers.get(requestId); - if (controller === undefined) { +export function abortRequest(requestId: string, expectedIncarnation?: string): boolean { + const entry = controllers.get(requestId); + if (entry === undefined || !matches(entry, expectedIncarnation)) { return false; } - controller.abort(); + entry.controller.abort(); return true; } @@ -45,8 +88,10 @@ export function deregisterAbortController(requestId: string): void { } /** - * Check whether a request has an active abort controller. + * Check whether a request has an active abort controller. With + * `expectedIncarnation`, only a controller of that incarnation counts. */ -export function hasActiveAbortController(requestId: string): boolean { - return controllers.has(requestId); +export function hasActiveAbortController(requestId: string, expectedIncarnation?: string): boolean { + const entry = controllers.get(requestId); + return entry !== undefined && matches(entry, expectedIncarnation); } diff --git a/packages/engine/src/execution/index.ts b/packages/engine/src/execution/index.ts index 3c63ef16a8..11caf4425d 100644 --- a/packages/engine/src/execution/index.ts +++ b/packages/engine/src/execution/index.ts @@ -19,7 +19,8 @@ export { abortRequest, hasActiveAbortController, registerAbortController, - deregisterAbortController + deregisterAbortController, + tagAbortController } from "./abort-registry"; export { detectInterruptedRequests, diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index e6b299725d..b4f3ff91b4 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -21,7 +21,13 @@ import type { BlockTraceItem, ContinuationItem, SuspensionItem, SuspensionResume import type { RuntimeItem } from "@flow-state-dev/core/items/internal"; import type { ResumeContext } from "@flow-state-dev/core/types"; import { createExecutionContext } from "../context/createExecutionContext"; -import { isSameSession, resolveLineageId, resolveSessionStorageKey, tenantMatches } from "../stores/scope-keys"; +import { + isSameSession, + resolveLineageId, + resolveRequestIncarnation, + resolveSessionStorageKey, + tenantMatches +} from "../stores/scope-keys"; import { canSpeak, canSpeakStream, getRequestSideChainPool } from "@flow-state-dev/core"; import { createExecutionLogContext, @@ -969,8 +975,14 @@ export async function runActionInternal< // loser is refused now, having written nothing; the winner's // `createExecutionContext` adopts this stub, as it adopts the host's // enqueue-time one on the queued path. + // + // The incarnation this run answers to is the admitted record's, or the one + // the claim wrote (a same-owner hand-off keeps the holder's). Its abort + // controller carries it, so an abort fenced on another request under this + // id cannot fire this run. + let runIncarnation: string; if (admittedRequest === undefined) { - await claimRequestRecord( + const claimed = await claimRequestRecord( options.stores, options.flow, createInitialRequestRecord( @@ -990,6 +1002,9 @@ export async function runActionInternal< Date.now() ) ); + runIncarnation = resolveRequestIncarnation(claimed); + } else { + runIncarnation = resolveRequestIncarnation(admittedRequest); } await registry.register({ @@ -1405,7 +1420,7 @@ export async function runActionInternal< // If anything above threw, the controller would never be registered. // If anything between here and the main try block throws, the outer // try/catch below cleans it up. - const abortController = registerAbortController(requestId); + const abortController = registerAbortController(requestId, runIncarnation); // First poll (FIX-1026). Placed here rather than beside the timer because a // poll before this line has no controller to fire and therefore cannot // deliver — it would be a guaranteed-useless store read on every request. diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 58182c4535..45fa405a48 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -36,6 +36,7 @@ export { filesystemStores, inMemoryStores, resolveOrgStorageKey, + resolveRequestIncarnation, resolveTraceMaxRequests, resolveUserStorageKey, runWithCAS, diff --git a/packages/engine/src/routes/abort-routes.ts b/packages/engine/src/routes/abort-routes.ts index 233f1747eb..b251b29b5e 100644 --- a/packages/engine/src/routes/abort-routes.ts +++ b/packages/engine/src/routes/abort-routes.ts @@ -2,6 +2,7 @@ * Abort route handler for cancelling in-flight requests. */ import type { StoreRegistry } from "../stores/types"; +import { resolveRequestIncarnation } from "../stores/scope-keys"; import type { ResolvedPrincipal } from "../transports/types"; import { abortRequest, @@ -47,18 +48,23 @@ export async function handleAbortRequest( return unknownRequestResponse(requestId); } + // The request the owner check above read. Everything below acts on it and + // on nothing else that later takes the id. + const incarnation = resolveRequestIncarnation(record); + // One atomic step: record the intent only while the request is still // running. A read-then-write cannot express this — the worker can commit a // terminal status between the two, and writing afterwards would restore an - // `in_progress` record over a finished one. Fenced to the record checked - // above by its `createdAt`: if the id was deleted and taken by someone else - // since, the write misses and the caller gets the unused-id answer. + // `in_progress` record over a finished one. Fenced to the checked request by + // its incarnation: if the id was deleted and taken by someone else since, + // the write misses and the caller gets the unused-id answer; if the owner's + // own retry handed the record off, the incarnation held and the write lands. const result = await ctx.stores.request.setFieldsIfStatus( requestId, { abortRequested: true }, ["in_progress"], Date.now(), - record.createdAt + incarnation ); if (result.status === undefined) { @@ -71,9 +77,11 @@ export async function handleAbortRequest( }); } - // Fire the in-memory controller if this is the same instance. - if (hasActiveAbortController(requestId)) { - abortRequest(requestId); + // Fire the in-memory controller if the checked request runs in this + // process. A controller of a later request under the id is not it; that + // request never received the intent, so it is left running. + if (hasActiveAbortController(requestId, incarnation)) { + abortRequest(requestId, incarnation); return new Response(null, { status: 204 }); } diff --git a/packages/engine/src/stores/filesystem/request-store.ts b/packages/engine/src/stores/filesystem/request-store.ts index 2ff6528cae..9d1abd7ee6 100644 --- a/packages/engine/src/stores/filesystem/request-store.ts +++ b/packages/engine/src/stores/filesystem/request-store.ts @@ -21,7 +21,7 @@ import { withRequestSourceDefault, withStoredAbortRequested } from "../shared"; -import { matchesOrgFilter, matchesTenantFilter } from "../scope-keys"; +import { matchesOrgFilter, matchesTenantFilter, resolveRequestIncarnation } from "../scope-keys"; import { compareRequestsForListing } from "../list-order"; import { pollEvents } from "../subscribe-helpers"; import { appendFile, readdir, readFile, rm, stat } from "node:fs/promises"; @@ -391,7 +391,7 @@ export class FilesystemRequestStore implements RequestStore { fields: ConditionalRequestFields, allowedStatuses: readonly RequestStatus[], updatedAt: number, - expectedCreatedAt?: number + expectedIncarnation?: string ): Promise { const { abortRequested, ...recordFields } = fields; let found: RequestStatus | undefined; @@ -407,7 +407,10 @@ export class FilesystemRequestStore implements RequestStore { await this.store.update(id, async (current) => { found = current.status; // Another record under the same id is not the one the caller checked. - if (expectedCreatedAt !== undefined && current.createdAt !== expectedCreatedAt) { + if ( + expectedIncarnation !== undefined && + resolveRequestIncarnation(current) !== expectedIncarnation + ) { otherRecord = true; return current; } diff --git a/packages/engine/src/stores/index.ts b/packages/engine/src/stores/index.ts index 773446f35d..00853e5d12 100644 --- a/packages/engine/src/stores/index.ts +++ b/packages/engine/src/stores/index.ts @@ -80,7 +80,8 @@ export type { export { resolveUserStorageKey, - resolveOrgStorageKey + resolveOrgStorageKey, + resolveRequestIncarnation } from "./scope-keys"; export type { IsolationFlow } from "./scope-keys"; diff --git a/packages/engine/src/stores/memory/request-store.ts b/packages/engine/src/stores/memory/request-store.ts index 2ed7c0009a..4ca1240e6d 100644 --- a/packages/engine/src/stores/memory/request-store.ts +++ b/packages/engine/src/stores/memory/request-store.ts @@ -23,7 +23,7 @@ import { withRequestSourceDefault, withStoredAbortRequested } from "../shared"; -import { matchesOrgFilter, matchesTenantFilter } from "../scope-keys"; +import { matchesOrgFilter, matchesTenantFilter, resolveRequestIncarnation } from "../scope-keys"; import { compareRequestsForListing } from "../list-order"; import { BoundedQueue } from "../../utils/bounded-queue"; import { StoreSubscriptionError } from "../../errors/store-subscription-error"; @@ -73,11 +73,14 @@ export class InMemoryRequestStore implements RequestStore { fields: ConditionalRequestFields, allowedStatuses: readonly RequestStatus[], updatedAt: number, - expectedCreatedAt?: number + expectedIncarnation?: string ): Promise { const current = this.records.get(id); if (current === undefined) return { applied: false, status: undefined }; - if (expectedCreatedAt !== undefined && current.createdAt !== expectedCreatedAt) { + if ( + expectedIncarnation !== undefined && + resolveRequestIncarnation(current) !== expectedIncarnation + ) { return { applied: false, status: undefined }; } if (!allowedStatuses.includes(current.status)) { diff --git a/packages/engine/src/stores/testing/request-store-conformance.ts b/packages/engine/src/stores/testing/request-store-conformance.ts index 777a836445..3ac66b0d66 100644 --- a/packages/engine/src/stores/testing/request-store-conformance.ts +++ b/packages/engine/src/stores/testing/request-store-conformance.ts @@ -690,34 +690,132 @@ export function createRequestStoreConformanceTests( }); }); - // The identity fence: a caller that checked who owns the record passes - // its `createdAt`, so the write cannot land on another request that took - // the id after that check. - it("reports a record with another createdAt as absent, and writes nothing", async () => { - await withStore(async (store) => { - const requestId = "req_cond_fenced"; - await seed(store, requestId); - const { createdAt } = (await store.get(requestId))!; + // The identity fence. A caller that checked who owns the record passes + // that record's incarnation, so the write lands only on the request the + // check read: never on another request that took the id afterwards, and + // never missing the checked one because a same-owner hand-off rewrote + // its `createdAt`. A record written before incarnations existed answers + // to `legacy_`, spelled out here rather than read from the + // engine's helper, so a store that states the rule in its own query + // language is held to the same string. + describe("identity fence (expectedIncarnation)", () => { + const stamped = ( + requestId: string, + incarnation: string | null | undefined, + createdAt: number + ): RequestRecord => ({ + ...makeRecord(requestId, "in_progress", []), + createdAt, + updatedAt: createdAt, + startedAtMs: createdAt, + ...(incarnation === undefined ? {} : { incarnation: incarnation as string }) + }); - const missed = await store.setFieldsIfStatus( + async function fenced(store: RequestStore, requestId: string, expected: string) { + return store.setFieldsIfStatus( requestId, { abortRequested: true }, ["in_progress"], Date.now(), - createdAt + 1 + expected ); - expect(missed).toEqual({ applied: false, status: undefined }); - expect(await store.isAbortRequested(requestId)).toBe(false); + } - const hit = await store.setFieldsIfStatus( - requestId, - { abortRequested: true }, - ["in_progress"], - Date.now(), - createdAt - ); - expect(hit).toEqual({ applied: true, status: "in_progress" }); - expect(await store.isAbortRequested(requestId)).toBe(true); + it("reports a request with another incarnation as absent, even with the same createdAt", async () => { + await withStore(async (store) => { + const requestId = "req_fence_same_ms"; + await store.set(requestId, stamped(requestId, "inc_newer", 1_000), "any"); + + const missed = await fenced(store, requestId, "inc_checked"); + expect(missed).toEqual({ applied: false, status: undefined }); + expect(await store.isAbortRequested(requestId)).toBe(false); + }); + }); + + it("applies to the same incarnation even when createdAt was rewritten", async () => { + await withStore(async (store) => { + const requestId = "req_fence_handed_off"; + // What a same-owner hand-off leaves: a new createdAt, the kept incarnation. + await store.set(requestId, stamped(requestId, "inc_kept", 2_000), "any"); + + const hit = await fenced(store, requestId, "inc_kept"); + expect(hit).toEqual({ applied: true, status: "in_progress" }); + expect(await store.isAbortRequested(requestId)).toBe(true); + }); + }); + + it("fences a legacy record on legacy_", async () => { + await withStore(async (store) => { + const requestId = "req_fence_legacy"; + await store.set(requestId, stamped(requestId, undefined, 1_000), "any"); + + expect(await fenced(store, requestId, "legacy_1001")).toEqual({ + applied: false, + status: undefined + }); + expect(await store.isAbortRequested(requestId)).toBe(false); + expect(await fenced(store, requestId, "legacy_1000")).toEqual({ + applied: true, + status: "in_progress" + }); + }); + }); + + it("fences a handed-off legacy record on the value it stores, not its new createdAt", async () => { + await withStore(async (store) => { + const requestId = "req_fence_legacy_handed_off"; + await store.set(requestId, stamped(requestId, "legacy_1000", 2_000), "any"); + + expect(await fenced(store, requestId, "legacy_2000")).toEqual({ + applied: false, + status: undefined + }); + expect(await fenced(store, requestId, "legacy_1000")).toEqual({ + applied: true, + status: "in_progress" + }); + }); + }); + + it("never matches a stamped request to a legacy fence", async () => { + await withStore(async (store) => { + const requestId = "req_fence_stamped_vs_legacy"; + await store.set(requestId, stamped(requestId, "inc_new", 1_000), "any"); + + expect(await fenced(store, requestId, "legacy_1000")).toEqual({ + applied: false, + status: undefined + }); + expect(await store.isAbortRequested(requestId)).toBe(false); + }); + }); + + it("treats a null incarnation as absent", async () => { + await withStore(async (store) => { + const requestId = "req_fence_null"; + await store.set(requestId, stamped(requestId, null, 1_000), "any"); + + expect(await fenced(store, requestId, "legacy_1000")).toEqual({ + applied: true, + status: "in_progress" + }); + }); + }); + + it("reports the terminal status, not absence, when the incarnation matches", async () => { + await withStore(async (store) => { + const requestId = "req_fence_terminal"; + await store.set( + requestId, + { ...stamped(requestId, "inc_done", 1_000), status: "completed" }, + "any" + ); + + expect(await fenced(store, requestId, "inc_done")).toEqual({ + applied: false, + status: "completed" + }); + }); }); }); diff --git a/packages/engine/src/stores/types.ts b/packages/engine/src/stores/types.ts index ba191fab33..05c88ca20d 100644 --- a/packages/engine/src/stores/types.ts +++ b/packages/engine/src/stores/types.ts @@ -712,24 +712,26 @@ export interface RequestStore extends DeltaStoreOps { * An empty `allowedStatuses` matches nothing, and an absent record is never * a match — see {@link ConditionalWriteResult} for the three outcomes. * - * `expectedCreatedAt`, when given, fences the write to the record a caller - * already read: a record at `id` with a different `createdAt` is a different - * request (the id was deleted and taken again), and is reported exactly as - * an absent one, `{ applied: false, status: undefined }`, with nothing - * written. A caller that checked who owns the record passes it, so the write - * cannot land on a record that check never saw. + * `expectedIncarnation`, when given, fences the write to the request a + * caller already read: a record at `id` whose incarnation differs is a + * different request (the id was deleted and taken again), and is reported + * exactly as an absent one, `{ applied: false, status: undefined }`, with + * nothing written. A caller that checked who owns the record passes that + * record's `resolveRequestIncarnation(record)`, so the write cannot land on a + * record that check never saw. * - * `createdAt` is not the request's identity: two records under one id can - * share a millisecond, and a same-owner hand-off rewrites it. The identity - * is {@link RequestRecord.incarnation}; moving this fence onto it is an open - * follow-up. + * The store compares it with the stored record's incarnation resolved the + * same way — its `incarnation`, or `legacy_` when that is absent or + * null — inside the same atomic step as the status check. Never compare + * `createdAt`: two requests under one id can share a millisecond, and a + * same-owner hand-off rewrites it while keeping the incarnation. */ setFieldsIfStatus( id: string, fields: ConditionalRequestFields, allowedStatuses: readonly RequestStatus[], updatedAt: number, - expectedCreatedAt?: number + expectedIncarnation?: string ): Promise; /** diff --git a/packages/engine/src/transports/host/createInboundTransportHost.ts b/packages/engine/src/transports/host/createInboundTransportHost.ts index c906381343..195807eb56 100644 --- a/packages/engine/src/transports/host/createInboundTransportHost.ts +++ b/packages/engine/src/transports/host/createInboundTransportHost.ts @@ -21,7 +21,12 @@ import { continueRequest as continueRequestImpl, type ContinueRequestResult } from "../../execution/request-continuation"; -import { isSameSession, resolveSessionStorageKey, tenantMatches } from "../../stores/scope-keys"; +import { + isSameSession, + resolveRequestIncarnation, + resolveSessionStorageKey, + tenantMatches +} from "../../stores/scope-keys"; import { isTerminalRequestStatus } from "../../stores/subscribe-helpers"; import { createInitialRequestRecord } from "../../context/initial-request-record"; import { @@ -40,7 +45,8 @@ import { } from "../../execution/logging"; import { deregisterAbortController, - registerAbortController + registerAbortController, + tagAbortController } from "../../execution/abort-registry"; import { generateId } from "../../utils/generate-id"; import { @@ -622,8 +628,9 @@ export function createInboundTransportHost( * overwriting, and a lost race against this same owner (a retry reusing its * id) keeps the existing record and re-stamps it, which is the * last-write-wins hand-off it always was. - * Resolves `true` once the entry is this dispatch's to keep warm and to - * remove on exit. + * Resolves once the entry is this dispatch's to keep warm and to remove on + * exit, with the incarnation of the request record it claimed (a hand-off + * keeps the holder's), so a controller registered before it can be tagged. * * @param admitted The session `admitOwnership` read and admitted, if any. */ @@ -632,14 +639,14 @@ export function createInboundTransportHost( dispatchEnvelope: DispatchEnvelope, admitted: SessionRecord | undefined, entry: Omit[0], "flowKind" | "flowId"> - ): Promise => { + ): Promise => { const record = createInitialRequestRecord( { ...dispatchEnvelope, flowKind: flow.kind, flowId: flow.id }, entry.startedAt ); // Owner-fenced on both axes: another flow instance's or another user's // record under this id is never overwritten or re-parented. - await claimRequestRecord(stores, flow, record); + const claimed = await claimRequestRecord(stores, flow, record); // A request under a child session moves the child's update time here, // where the request is first recorded as working, and not when its run // starts: the run can wait behind a concurrency key, or in an external @@ -684,6 +691,7 @@ export function createInboundTransportHost( } } await stores.activeRequests.register({ ...entry, flowKind: flow.kind, flowId: flow.id }); + return resolveRequestIncarnation(claimed); }; const dispatch = (envelope: InboundRequestEnvelope): DispatchHandle => { @@ -926,7 +934,10 @@ export function createInboundTransportHost( lastHeartbeatAt: ts }) ) - .then(() => { + .then((incarnation) => { + // Until its record is claimed this controller belongs to no known + // request, so an abort fenced on one does not fire it; now it can. + tagAbortController(requestId, queuedAbort, incarnation); entryOwned = true; }) .catch(async (error: unknown) => { diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index d27c97de3b..bd198eb21b 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -1,13 +1,15 @@ import { defineFlow, handler, sequencer } from "@flow-state-dev/core"; import { DEFAULT_ORG_ID } from "@flow-state-dev/core"; import { z } from "zod"; -import { describe, expect, it, beforeEach } from "vitest"; +import { afterEach, describe, expect, it, beforeEach } from "vitest"; import { continueRequest, createInMemoryStores, runAction, createFlowRegistry, - createFlowApiRouter + createFlowApiRouter, + createInboundTransportHost, + defaultBodyUserIdPrincipalResolver } from "../src"; import { createCheckpointDurabilityProvider } from "../src/durability/checkpoint-durability-provider"; import { parseFlowRoute } from "../src/routes/parseFlowRoute"; @@ -15,10 +17,13 @@ import { registerAbortController, abortRequest, deregisterAbortController, - hasActiveAbortController + hasActiveAbortController, + tagAbortController } from "../src/execution/abort-registry"; import { handleAbortRequest } from "../src/routes/abort-routes"; -import type { StoreRegistry } from "../src/stores/types"; +import type { RequestRecord, StoreRegistry } from "../src/stores/types"; +import { createInitialRequestRecord } from "../src/context/initial-request-record"; +import { claimRequestRecord } from "../src/context/request-principal"; // --------------------------------------------------------------------------- // Abort registry unit tests @@ -66,6 +71,53 @@ describe("abort-registry", () => { const result = abortRequest("test-req-1"); expect(result).toBe(true); }); + + // An id can name two requests over time, so a fire that names the request + // it means acts only on that request's controller. + describe("fenced by incarnation", () => { + it("fires only the controller of the expected incarnation", () => { + const controller = registerAbortController("test-req-1", "inc_a"); + + expect(hasActiveAbortController("test-req-1", "inc_b")).toBe(false); + expect(abortRequest("test-req-1", "inc_b")).toBe(false); + expect(controller.signal.aborted).toBe(false); + + expect(hasActiveAbortController("test-req-1", "inc_a")).toBe(true); + expect(abortRequest("test-req-1", "inc_a")).toBe(true); + expect(controller.signal.aborted).toBe(true); + }); + + it("does not fire an untagged controller for a fenced abort", () => { + const controller = registerAbortController("test-req-1"); + + expect(abortRequest("test-req-1", "inc_a")).toBe(false); + expect(controller.signal.aborted).toBe(false); + }); + + it("fires a controller once it is tagged with its request's incarnation", () => { + const controller = registerAbortController("test-req-1"); + tagAbortController("test-req-1", controller, "inc_a"); + + expect(abortRequest("test-req-1", "inc_a")).toBe(true); + expect(controller.signal.aborted).toBe(true); + }); + + it("does not tag a controller that has since been replaced", () => { + const first = registerAbortController("test-req-1"); + const second = registerAbortController("test-req-1", "inc_b"); + tagAbortController("test-req-1", first, "inc_a"); + + expect(abortRequest("test-req-1", "inc_a")).toBe(false); + expect(second.signal.aborted).toBe(false); + }); + + it("still fires by id alone when no incarnation is expected", () => { + const controller = registerAbortController("test-req-1", "inc_a"); + + expect(abortRequest("test-req-1")).toBe(true); + expect(controller.signal.aborted).toBe(true); + }); + }); }); // --------------------------------------------------------------------------- @@ -113,7 +165,8 @@ describe("handleAbortRequest", () => { }); it("returns 204 when aborting an active request with in-memory controller", async () => { - registerAbortController("req_active"); + // The run's controller carries the incarnation of the request it runs. + registerAbortController("req_active", "inc_active"); await stores.request.set("req_active", { orgId: DEFAULT_ORG_ID, id: "req_active", @@ -126,6 +179,7 @@ describe("handleAbortRequest", () => { version: 1, createdAt: Date.now(), updatedAt: Date.now(), + incarnation: "inc_active", journal: [] } as any, "any"); @@ -266,6 +320,108 @@ describe("handleAbortRequest", () => { expect(stored?.abortRequested).not.toBe(true); }); + // The fence names the request by its incarnation, not its `createdAt`. + // These interleaves are the ones `createdAt` gets wrong in each direction, + // so both records share or change `createdAt` exactly where it misleads. + describe("fenced on the request's incarnation", () => { + const REUSED = "req_fence_incarnation"; + const flow = defineFlow({ + kind: "chat", + actions: { + run: { + inputSchema: z.unknown(), + block: handler({ name: "noop", inputSchema: z.unknown(), execute: () => ({}) }) + } + } + })({ id: "chat" }); + const record = (tenantId: string | undefined, at: number): RequestRecord => + createInitialRequestRecord( + { requestId: REUSED, flowKind: "chat", flowId: "chat", actionName: "run", userId: "user_1", tenantId }, + at + ); + + /** Run the route, doing `between` after its owner-check read and before its write. */ + async function abortWithInterleave( + tenantId: string | undefined, + between: () => Promise + ): Promise { + const read = stores.request.get.bind(stores.request); + stores.request.get = async (id: string) => { + const seen = await read(id); + await between(); + return seen; + }; + try { + const response = await handleAbortRequest( + new Request(`http://localhost/api/flows/chat/requests/${REUSED}/abort`, { method: "POST" }), + { kind: "abort_request", flowKind: "chat", requestId: REUSED }, + { stores, tenantId } + ); + return response.status; + } finally { + stores.request.get = read; + } + } + + afterEach(() => deregisterAbortController(REUSED)); + + it("misses another tenant's request that took the id in the same millisecond", async () => { + await stores.request.set(REUSED, record("tenant_a", 1_000), "absent"); + const other = record("tenant_b", 1_000); + + const status = await abortWithInterleave("tenant_a", async () => { + await stores.request.delete(REUSED); + await stores.request.set(REUSED, other, "absent"); + }); + + expect(status).toBe(404); + const stored = (await stores.request.get(REUSED))!; + expect(stored.incarnation).toBe(other.incarnation); + expect(stored.abortRequested).not.toBe(true); + }); + + it("records the cancel on the owner's request after their own retry handed it off", async () => { + await claimRequestRecord(stores, flow, record(undefined, 1_000)); + const before = (await stores.request.get(REUSED))!; + + const status = await abortWithInterleave(undefined, async () => { + await claimRequestRecord(stores, flow, record(undefined, 2_000)); + }); + + const stored = (await stores.request.get(REUSED))!; + // The hand-off really did rewrite `createdAt`; only the incarnation held. + expect(stored.createdAt).not.toBe(before.createdAt); + expect(stored.incarnation).toBe(before.incarnation); + expect(status).toBe(202); + expect(stored.abortRequested).toBe(true); + }); + + it("does not fire a later request's controller in this process", async () => { + // The write landed on the request the check read; before the route + // fires locally, that request finished and a later one under the id + // started here. Its controller carries its own incarnation. + const checked = record(undefined, 1_000); + await stores.request.set(REUSED, checked, "absent"); + const later = registerAbortController(REUSED, "inc_later_request"); + + const status = await abortWithInterleave(undefined, async () => {}); + + expect(later.signal.aborted).toBe(false); + expect(status).toBe(202); + }); + + it("fires the controller of the request it fenced on", async () => { + const checked = record(undefined, 1_000); + await stores.request.set(REUSED, checked, "absent"); + const own = registerAbortController(REUSED, checked.incarnation); + + const status = await abortWithInterleave(undefined, async () => {}); + + expect(own.signal.aborted).toBe(true); + expect(status).toBe(204); + }); + }); + it("returns 409 when request is already completed", async () => { // Seed a completed request record await stores.request.set("req_terminal", { @@ -1586,7 +1742,10 @@ describe("handleAbortRequest — conditional write", () => { const stores = createInMemoryStores(); const requestId = "req_cond_local"; await seedRecord(stores, requestId, "in_progress"); - const controller = registerAbortController(requestId); + // A record with no stored incarnation answers to `legacy_`, + // and the run's controller carries that same value. + const seeded = (await stores.request.get(requestId))!; + const controller = registerAbortController(requestId, `legacy_${seeded.createdAt}`); const response = await postAbort(stores, requestId); @@ -2317,3 +2476,99 @@ describe("cross-process abort delivered during the background drain", () => { expect((await stores.request.get(requestId))?.status).toBe("completed"); }); }); + +// A request queued behind a concurrency key is cancellable before its run +// registers anything: the host registers its controller at dispatch, before the +// request record exists, and tags it with the record's incarnation once the +// record is claimed. The route's fenced fire then reaches it and the queued +// run never starts. +describe("a queued request cancelled through the route", () => { + it("fires the host's queued controller once its record is claimed", async () => { + const stores = createInMemoryStores(); + const executed: string[] = []; + let releaseHold!: () => void; + const hold = new Promise((resolve) => { + releaseHold = resolve; + }); + let markHolding!: () => void; + const holding = new Promise((resolve) => { + markHolding = resolve; + }); + const concurrency = { policy: "queue", key: "session" } as const; + const registry = createFlowRegistry(); + registry.register( + defineFlow({ + kind: "queued-abort", + actions: { + hold: { + inputSchema: z.object({}), + concurrency, + block: handler({ + name: "hold", + inputSchema: z.object({}), + execute: async () => { + markHolding(); + await hold; + return {}; + } + }) + }, + queued: { + inputSchema: z.object({}), + concurrency, + block: handler({ + name: "queued", + inputSchema: z.object({}), + execute: () => { + executed.push("queued"); + return {}; + } + }) + } + } + })({ id: "queued-abort" }) + ); + const host = createInboundTransportHost({ + registry, + stores, + resolvePrincipal: defaultBodyUserIdPrincipalResolver, + runtimeConfig: {} + }); + const dispatch = (action: "hold" | "queued", requestId: string) => + host.dispatch({ + source: "http", + flowKind: "queued-abort", + action, + input: {}, + sessionId: "s_queued_abort", + requestId, + orgId: DEFAULT_ORG_ID, + principal: { userId: "u_queued", orgId: DEFAULT_ORG_ID } + }); + + const holder = dispatch("hold", "req_queued_holder"); + await holding; + const queued = dispatch("queued", "req_queued_waiting"); + // Wait for the enqueue-time record, then let the claim's continuation run. + for (let i = 0; i < 100 && (await stores.request.get("req_queued_waiting")) === undefined; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await new Promise((resolve) => setTimeout(resolve, 20)); + + const response = await handleAbortRequest( + new Request("http://localhost/api/flows/queued-abort/requests/req_queued_waiting/abort", { + method: "POST" + }), + { kind: "abort_request", flowKind: "queued-abort", requestId: "req_queued_waiting" }, + { stores } + ); + + releaseHold(); + await holder.finished; + await queued.finished.catch(() => undefined); + + // 204: fired here, on the controller the host registered and tagged. + expect(response.status).toBe(204); + expect(executed).toEqual([]); + }); +}); diff --git a/packages/integration-tests/src/scenarios/abort-fenced-on-incarnation.test.ts b/packages/integration-tests/src/scenarios/abort-fenced-on-incarnation.test.ts new file mode 100644 index 0000000000..b036cd7c1b --- /dev/null +++ b/packages/integration-tests/src/scenarios/abort-fenced-on-incarnation.test.ts @@ -0,0 +1,219 @@ +/** + * A cancel lands on the request its caller checked, and on nothing else that + * holds the id. + * + * The abort route reads the record, checks the caller owns it, and then writes + * the intent. Between the read and the write the id can change hands in two + * ways, and the route must answer each one correctly: + * + * (a) The owner's own retry re-stamps the record under the same id. That is + * still the owner's request: same incarnation, rewritten `createdAt`. + * The cancel must land and the run must end `aborted`. + * (b) The checked request finished, its record went, and a later request + * took the id. That request never received this cancel: it must keep + * running, and its record must not carry the intent. + * + * Both legs run a real flow through `runAction` on one shared store and cancel + * it through the flow router's abort route, in the same process as the run, so + * the route's in-process fire is on the path too. The interleave is staged by + * wrapping the store's `get` for the one call the route's owner check makes. + * + * The give-up timer in the parked block is the falsifier: a cancel that never + * arrives surfaces as a `completed` run rather than a hung test. + */ +import { describe, expect, it } from "vitest"; +import { DEFAULT_ORG_ID, defineFlow, handler } from "@flow-state-dev/core"; +import type { BlockContext } from "@flow-state-dev/core/types"; +import { + createFlowApiRouter, + createFlowRegistry, + createInMemoryStores, + runAction +} from "@flow-state-dev/engine"; +import type { RequestRecord, StoreRegistry } from "@flow-state-dev/engine"; +import { createMockModelResolver } from "@flow-state-dev/testing"; +import { z } from "zod"; + +const FLOW_KIND = "abort-fence"; +const USER_ID = "u_fence"; + +type Park = { + /** Resolves once the run is inside its block, so the cancel meets a live run. */ + started: Promise; + /** Whether the run's own signal fired before its give-up timer. */ + abortedWhenDone?: boolean; +}; + +/** + * A block that parks until its signal fires or `selfCompleteMs` passes, and + * throws on abort so the run settles `aborted` the way a real cancelled step + * does. + */ +function parkingFlow(park: Park & { markStarted: () => void }, selfCompleteMs: number) { + return defineFlow({ + kind: FLOW_KIND, + actions: { + run: { + inputSchema: z.unknown(), + block: handler({ + name: "park", + inputSchema: z.unknown(), + outputSchema: z.string(), + execute: async (_input: unknown, ctx: BlockContext) => { + park.markStarted(); + const outcome = await new Promise<"aborted" | "self-completed">((resolve) => { + if (ctx.signal?.aborted === true) return resolve("aborted"); + const timer = setTimeout(() => resolve("self-completed"), selfCompleteMs); + ctx.signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve("aborted"); + }, + { once: true } + ); + }); + park.abortedWhenDone = outcome === "aborted"; + if (outcome === "aborted") throw new DOMException("Aborted", "AbortError"); + return outcome; + } + }) + } + } + })({ id: FLOW_KIND }); +} + +function newPark(): Park & { markStarted: () => void } { + let markStarted!: () => void; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + return { started, markStarted }; +} + +function run(flow: ReturnType, stores: StoreRegistry, requestId: string) { + return runAction({ + flow, + actionName: "run", + input: {}, + requestId, + userId: USER_ID, + orgId: DEFAULT_ORG_ID, + stores, + runtimeConfig: { modelResolver: createMockModelResolver({}) } + }); +} + +/** + * POST the flow router's abort route. The route's owner-check read of + * `requestId` returns `read(fresh)`, and `between` runs after that read and + * before the route writes anything. + */ +async function abortThroughRouter( + flow: ReturnType, + stores: StoreRegistry, + requestId: string, + interleave: { + read?: (fresh: RequestRecord | undefined) => RequestRecord | undefined; + between?: () => Promise; + } +): Promise { + const registry = createFlowRegistry(); + registry.register(flow); + const router = createFlowApiRouter({ registry, stores, detectInterruptedOnStartup: false }); + + const get = stores.request.get.bind(stores.request); + let intercepted = false; + stores.request.get = async (id: string) => { + const fresh = await get(id); + if (intercepted || id !== requestId) return fresh; + intercepted = true; + const seen = interleave.read === undefined ? fresh : interleave.read(fresh); + await interleave.between?.(); + return seen; + }; + try { + const response = await router.POST( + new Request(`http://localhost/api/flows/${FLOW_KIND}/requests/${requestId}/abort`, { + method: "POST" + }), + { params: { path: [FLOW_KIND, "requests", requestId, "abort"] } } + ); + return response.status; + } finally { + stores.request.get = get; + } +} + +describe("a cancel is fenced on the request its caller checked", () => { + it("(a) stops the owner's run after their own retry re-stamped its record", async () => { + const stores = createInMemoryStores(); + const requestId = "req_fence_handoff"; + const park = newPark(); + const flow = parkingFlow(park, 3_000); + + const running = run(flow, stores, requestId); + await park.started; + const before = (await stores.request.get(requestId))!; + + const status = await abortThroughRouter(flow, stores, requestId, { + // The owner's retry under the same id: the same request, re-stamped + // with a later `createdAt` and its incarnation kept. + between: async () => { + const current = (await stores.request.get(requestId))!; + await stores.request.set( + requestId, + { ...current, createdAt: current.createdAt + 1_000 }, + "any" + ); + } + }); + + await running; + const record = (await stores.request.get(requestId))!; + + // Precondition: the hand-off really rewrote `createdAt`, and kept the + // incarnation, so only an identity fence can recognise the request. + expect(record.createdAt).not.toBe(before.createdAt); + expect(record.incarnation).toBe(before.incarnation); + + expect(status).toBe(204); + expect(park.abortedWhenDone).toBe(true); + expect(record.status).toBe("aborted"); + expect(record.abortRequested).toBe(true); + }); + + it("(b) leaves a later request under the same id running and unmarked", async () => { + const stores = createInMemoryStores(); + const requestId = "req_fence_reused"; + + // The request the caller checks: captured while it runs, then finished. + const firstPark = newPark(); + const first = run(parkingFlow(firstPark, 0), stores, requestId); + await firstPark.started; + const checked = (await stores.request.get(requestId))!; + expect(checked.status).toBe("in_progress"); + await first; + + // Its record goes, and a later request by the same owner takes the id. + await stores.request.delete(requestId); + const laterPark = newPark(); + const flow = parkingFlow(laterPark, 300); + const later = run(flow, stores, requestId); + await laterPark.started; + + // The route's owner check read the checked request before it finished. + const status = await abortThroughRouter(flow, stores, requestId, { + read: () => checked + }); + + await later; + const record = (await stores.request.get(requestId))!; + + expect(record.incarnation).not.toBe(checked.incarnation); + expect(status).toBe(404); + expect(laterPark.abortedWhenDone).toBe(false); + expect(record.status).toBe("completed"); + expect(record.abortRequested).not.toBe(true); + }); +}); diff --git a/packages/store-postgres/src/request-store.ts b/packages/store-postgres/src/request-store.ts index 741e2c01cf..3f43710884 100644 --- a/packages/store-postgres/src/request-store.ts +++ b/packages/store-postgres/src/request-store.ts @@ -377,7 +377,7 @@ export function createPostgresRequestStore( fields: ConditionalRequestFields, allowedStatuses: readonly RequestStatus[], updatedAt: number, - expectedCreatedAt?: number + expectedIncarnation?: string ): Promise { // ONE statement, and the predicate read is a LOCKING one. `locked` // takes a row lock and — unlike a plain read, which is pinned to the @@ -416,7 +416,8 @@ export function createPostgresRequestStore( // carry the old value back and move the indexed column BACKWARD. const result = await executor.query( `WITH locked AS ( - SELECT id, status, (data->>'createdAt')::bigint AS created_at + SELECT id, status, + COALESCE(data->>'incarnation', 'legacy_' || (data->>'createdAt')) AS incarnation FROM requests WHERE id = $1 FOR UPDATE ), applied AS ( @@ -425,23 +426,23 @@ export function createPostgresRequestStore( updated_at = $3 FROM locked l WHERE r.id = l.id AND l.status = ANY($4::text[]) - AND ($5::bigint IS NULL OR l.created_at = $5::bigint) + AND ($5::text IS NULL OR l.incarnation = $5::text) RETURNING r.id ) - SELECT l.status AS status, l.created_at AS created_at, + SELECT l.status AS status, l.incarnation AS incarnation, EXISTS (SELECT 1 FROM applied) AS applied FROM locked l`, - [id, JSON.stringify(fields), updatedAt, [...allowedStatuses], expectedCreatedAt ?? null] + [id, JSON.stringify(fields), updatedAt, [...allowedStatuses], expectedIncarnation ?? null] ); // No row means no record: `rows`, not `rowCount` — a PGlite-backed // executor reports `affectedRows` there, which is 0 for a SELECT. const row = result.rows[0] as - | { status: RequestStatus; created_at: string | number; applied: boolean } + | { status: RequestStatus; incarnation: string; applied: boolean } | undefined; if (row === undefined) return { applied: false, status: undefined }; // Another record under the same id is not the one the caller checked. - if (expectedCreatedAt !== undefined && Number(row.created_at) !== expectedCreatedAt) { + if (expectedIncarnation !== undefined && row.incarnation !== expectedIncarnation) { return { applied: false, status: undefined }; } return { applied: row.applied === true, status: row.status }; diff --git a/packages/store-sqlite/src/request-store.ts b/packages/store-sqlite/src/request-store.ts index 278f49c01c..0120b20c03 100644 --- a/packages/store-sqlite/src/request-store.ts +++ b/packages/store-sqlite/src/request-store.ts @@ -391,7 +391,7 @@ export function createSQLiteRequestStore( fields: ConditionalRequestFields, allowedStatuses: readonly RequestStatus[], updatedAt: number, - expectedCreatedAt?: number + expectedIncarnation?: string ): Promise { // One transaction, and it must be IMMEDIATE. A plain better-sqlite3 // transaction is DEFERRED: it takes only a read lock at the first @@ -411,7 +411,12 @@ export function createSQLiteRequestStore( if (row === undefined) return { applied: false, status: undefined }; const record = JSON.parse(row.data) as RequestRecord; // Another record under the same id is not the one the caller checked. - if (expectedCreatedAt !== undefined && record.createdAt !== expectedCreatedAt) { + // The incarnation rule is the engine's `resolveRequestIncarnation`, + // restated here because this package takes only types from the engine. + if ( + expectedIncarnation !== undefined && + (record.incarnation ?? `legacy_${record.createdAt}`) !== expectedIncarnation + ) { return { applied: false, status: undefined }; } if (!allowedStatuses.includes(row.status)) { From dd940f9b9a81a1e24b54ec81fbed360405e5e36f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 02:21:32 +0000 Subject: [PATCH 02/12] fix(engine): tag the queued abort controller as soon as the record is claimed A cancel can land once the enqueue-time record is claimed, while the host is still materializing the rest of the request. Tagging only after materialization left the controller unfenced for that window, so the route answered 202 and, with heartbeats off, the cancelled request ran. materializeOwned now reports the claimed incarnation through a callback right after the claim. Review nits: the route branches 204/202 on abortRequest's return value; runAction resolves its incarnation in one const; tagAbortController stays off the public barrel and the changeset. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- .changeset/abort-fence-incarnation.md | 2 +- packages/engine/src/execution/index.ts | 3 +- packages/engine/src/execution/runAction.ts | 49 ++++++------- packages/engine/src/routes/abort-routes.ts | 8 +-- .../host/createInboundTransportHost.ts | 24 ++++--- packages/engine/test/abort.test.ts | 71 +++++++++++++++++++ 6 files changed, 113 insertions(+), 44 deletions(-) diff --git a/.changeset/abort-fence-incarnation.md b/.changeset/abort-fence-incarnation.md index c9ce2d429c..0b15c4c914 100644 --- a/.changeset/abort-fence-incarnation.md +++ b/.changeset/abort-fence-incarnation.md @@ -6,4 +6,4 @@ A cancel now lands on the request its caller checked, identified by incarnation **Store authors:** `RequestStore.setFieldsIfStatus`'s fifth parameter is now `expectedIncarnation?: string`, replacing `expectedCreatedAt?: number`. Compare it with the stored record's `resolveRequestIncarnation(record)` (newly exported) inside the same atomic step as the status check, and report a mismatch as an absent record. See the engine README, "Abort intent on `RequestStore`". -The in-process abort registry takes the incarnation too: `registerAbortController(id, incarnation?)`, `abortRequest(id, expectedIncarnation?)` and `hasActiveAbortController(id, expectedIncarnation?)`, plus a new `tagAbortController(id, controller, incarnation)`. Callers that pass no incarnation behave as before. +The in-process abort registry takes the incarnation too: `registerAbortController(id, incarnation?)`, `abortRequest(id, expectedIncarnation?)` and `hasActiveAbortController(id, expectedIncarnation?)`. Callers that pass no incarnation behave as before. diff --git a/packages/engine/src/execution/index.ts b/packages/engine/src/execution/index.ts index 11caf4425d..3c63ef16a8 100644 --- a/packages/engine/src/execution/index.ts +++ b/packages/engine/src/execution/index.ts @@ -19,8 +19,7 @@ export { abortRequest, hasActiveAbortController, registerAbortController, - deregisterAbortController, - tagAbortController + deregisterAbortController } from "./abort-registry"; export { detectInterruptedRequests, diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index b4f3ff91b4..d8ce5e26f6 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -980,32 +980,29 @@ export async function runActionInternal< // the claim wrote (a same-owner hand-off keeps the holder's). Its abort // controller carries it, so an abort fenced on another request under this // id cannot fire this run. - let runIncarnation: string; - if (admittedRequest === undefined) { - const claimed = await claimRequestRecord( - options.stores, - options.flow, - createInitialRequestRecord( - { - requestId, - flowKind: options.flow.kind, - flowId: options.flow.id, - actionName: options.actionName as string, - userId: options.userId, - sessionId: options.sessionId, - tenantId: options.tenantId, - orgId: options.orgId, - source, - metadata: options.metadata, - input: options.input - }, - Date.now() - ) - ); - runIncarnation = resolveRequestIncarnation(claimed); - } else { - runIncarnation = resolveRequestIncarnation(admittedRequest); - } + const runIncarnation = resolveRequestIncarnation( + admittedRequest ?? + (await claimRequestRecord( + options.stores, + options.flow, + createInitialRequestRecord( + { + requestId, + flowKind: options.flow.kind, + flowId: options.flow.id, + actionName: options.actionName as string, + userId: options.userId, + sessionId: options.sessionId, + tenantId: options.tenantId, + orgId: options.orgId, + source, + metadata: options.metadata, + input: options.input + }, + Date.now() + ) + )) + ); await registry.register({ requestId, diff --git a/packages/engine/src/routes/abort-routes.ts b/packages/engine/src/routes/abort-routes.ts index b251b29b5e..cd9dec7bb2 100644 --- a/packages/engine/src/routes/abort-routes.ts +++ b/packages/engine/src/routes/abort-routes.ts @@ -4,10 +4,7 @@ import type { StoreRegistry } from "../stores/types"; import { resolveRequestIncarnation } from "../stores/scope-keys"; import type { ResolvedPrincipal } from "../transports/types"; -import { - abortRequest, - hasActiveAbortController -} from "../execution/abort-registry"; +import { abortRequest } from "../execution/abort-registry"; import { callerReachesRequest, jsonResponse, unknownRequestResponse } from "./route-utils"; import type { ParsedFlowRoute } from "./parseFlowRoute"; @@ -80,8 +77,7 @@ export async function handleAbortRequest( // Fire the in-memory controller if the checked request runs in this // process. A controller of a later request under the id is not it; that // request never received the intent, so it is left running. - if (hasActiveAbortController(requestId, incarnation)) { - abortRequest(requestId, incarnation); + if (abortRequest(requestId, incarnation)) { return new Response(null, { status: 204 }); } diff --git a/packages/engine/src/transports/host/createInboundTransportHost.ts b/packages/engine/src/transports/host/createInboundTransportHost.ts index 195807eb56..f96c2b2bec 100644 --- a/packages/engine/src/transports/host/createInboundTransportHost.ts +++ b/packages/engine/src/transports/host/createInboundTransportHost.ts @@ -629,17 +629,21 @@ export function createInboundTransportHost( * id) keeps the existing record and re-stamps it, which is the * last-write-wins hand-off it always was. * Resolves once the entry is this dispatch's to keep warm and to remove on - * exit, with the incarnation of the request record it claimed (a hand-off - * keeps the holder's), so a controller registered before it can be tagged. + * exit. * * @param admitted The session `admitOwnership` read and admitted, if any. + * @param onClaimed Called with the claimed record's incarnation (a hand-off + * keeps the holder's) the moment the record is claimed, before any later + * await: from then on the record is visible and cancellable, so a + * controller registered before it must carry the incarnation by then. */ const materializeOwned = async ( flow: FlowInstance, dispatchEnvelope: DispatchEnvelope, admitted: SessionRecord | undefined, - entry: Omit[0], "flowKind" | "flowId"> - ): Promise => { + entry: Omit[0], "flowKind" | "flowId">, + onClaimed?: (incarnation: string) => void + ): Promise => { const record = createInitialRequestRecord( { ...dispatchEnvelope, flowKind: flow.kind, flowId: flow.id }, entry.startedAt @@ -647,6 +651,7 @@ export function createInboundTransportHost( // Owner-fenced on both axes: another flow instance's or another user's // record under this id is never overwritten or re-parented. const claimed = await claimRequestRecord(stores, flow, record); + onClaimed?.(resolveRequestIncarnation(claimed)); // A request under a child session moves the child's update time here, // where the request is first recorded as working, and not when its run // starts: the run can wait behind a concurrency key, or in an external @@ -691,7 +696,6 @@ export function createInboundTransportHost( } } await stores.activeRequests.register({ ...entry, flowKind: flow.kind, flowId: flow.id }); - return resolveRequestIncarnation(claimed); }; const dispatch = (envelope: InboundRequestEnvelope): DispatchHandle => { @@ -932,12 +936,14 @@ export function createInboundTransportHost( metadata: dispatchEnvelope.metadata, startedAt: ts, lastHeartbeatAt: ts + }, (incarnation) => { + // Until its record is claimed this controller belongs to no known + // request, so an abort fenced on one does not fire it. Tagged the + // moment the record exists, because a cancel can land from then on. + tagAbortController(requestId, queuedAbort, incarnation); }) ) - .then((incarnation) => { - // Until its record is claimed this controller belongs to no known - // request, so an abort fenced on one does not fire it; now it can. - tagAbortController(requestId, queuedAbort, incarnation); + .then(() => { entryOwned = true; }) .catch(async (error: unknown) => { diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index bd198eb21b..6c20e6585b 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -2571,4 +2571,75 @@ describe("a queued request cancelled through the route", () => { expect(response.status).toBe(204); expect(executed).toEqual([]); }); + + // With heartbeats off nothing polls the stored intent later, so the fire is + // the only delivery. A cancel can land as soon as the record is claimed, while + // the host is still materializing the rest of the request; the controller has + // to carry the incarnation by then, or the fenced fire skips it and the + // cancelled request runs. + it("fires it for a cancel that lands while the host is still materializing, with heartbeats off", async () => { + const stores = createInMemoryStores(); + const executed: string[] = []; + const registry = createFlowRegistry(); + registry.register( + defineFlow({ + kind: "queued-abort-mid", + request: { heartbeatIntervalMs: 0 }, + actions: { + queued: { + inputSchema: z.object({}), + concurrency: { policy: "queue", key: "session" }, + block: handler({ + name: "queued", + inputSchema: z.object({}), + execute: () => { + executed.push("queued"); + return {}; + } + }) + } + } + })({ id: "queued-abort-mid" }) + ); + const requestId = "req_queued_mid"; + + // The last materialization step, after the claim: cancel from inside it. + let status: number | undefined; + const register = stores.activeRequests.register.bind(stores.activeRequests); + stores.activeRequests.register = async (entry) => { + if (entry.requestId === requestId && status === undefined) { + const response = await handleAbortRequest( + new Request(`http://localhost/api/flows/queued-abort-mid/requests/${requestId}/abort`, { + method: "POST" + }), + { kind: "abort_request", flowKind: "queued-abort-mid", requestId }, + { stores } + ); + status = response.status; + } + return register(entry); + }; + + const host = createInboundTransportHost({ + registry, + stores, + resolvePrincipal: defaultBodyUserIdPrincipalResolver, + runtimeConfig: {} + }); + const handle = host.dispatch({ + source: "http", + flowKind: "queued-abort-mid", + action: "queued", + input: {}, + sessionId: "s_queued_mid", + requestId, + orgId: DEFAULT_ORG_ID, + principal: { userId: "u_queued", orgId: DEFAULT_ORG_ID } + }); + await handle.finished.catch(() => undefined); + + expect(status).toBe(204); + expect(executed).toEqual([]); + expect((await stores.request.get(requestId))?.status).toBe("aborted"); + }); }); From c323625a750a844400bac32991b0295dbdc249f5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 02:32:24 +0000 Subject: [PATCH 03/12] fix(engine): re-tag the run's abort controller with the request it adopts runAction tagged its controller from admission's read, but the execution context reads the record again and runs as whatever request holds the id then. When another request took the id in between, a cancel fenced on the adopted request missed this run's controller, and with heartbeats off the run kept going. The controller is now re-tagged with the adopted incarnation, and a cancel recorded while the tag was stale is delivered from one read of the stored intent. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- packages/engine/src/execution/runAction.ts | 16 +++- packages/engine/test/abort.test.ts | 90 ++++++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index d8ce5e26f6..289b3c128c 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -61,7 +61,8 @@ import { generateId } from "../utils/generate-id"; import { abortRequest, registerAbortController, - deregisterAbortController + deregisterAbortController, + tagAbortController } from "./abort-registry"; import { assertSessionAdmitted, @@ -1630,6 +1631,19 @@ export async function runActionInternal< effectiveRuntimeConfig: options.runtimeConfig }); + // The controller was tagged from admission's read, but the context reads + // the record again and runs as whatever request holds the id now. If + // another request took the id in between, re-tag the controller with the + // one this run executes as, so a cancel fenced on it fires here. A cancel + // of that request recorded while the tag was still the earlier one missed + // the controller, so read the stored intent once and deliver it. + if (ctx.request.incarnation !== runIncarnation) { + tagAbortController(requestId, abortController, ctx.request.incarnation); + if (await options.stores.request.isAbortRequested(requestId).catch(() => false)) { + abortController.abort(); + } + } + // Resume mode: load the suspension record + checkpoint to restore the durable // sequencer's accumulator state. `resumeOf` (legacy two-request path) reads // from the ORIGINAL request id; same-request replay (FIX-811) reads from this diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index 6c20e6585b..e15416886c 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -2643,3 +2643,93 @@ describe("a queued request cancelled through the route", () => { expect((await stores.request.get(requestId))?.status).toBe("aborted"); }); }); + +// The run's controller must carry the incarnation of the request the run +// actually executes as. Admission and the execution context read the record +// separately, and a request that took the id in between is the one the context +// adopts; a cancel fenced on that request has to reach this run. Heartbeats are +// off, so the route's fire is the only delivery. +describe("a run's abort controller carries the incarnation it executes as", () => { + it("fires for a cancel of the request the context adopted, not the one admission read", async () => { + const stores = createInMemoryStores(); + const requestId = "req_adopted_later"; + let status: number | undefined; + const flow = defineFlow({ + kind: "adopt-fence", + request: { heartbeatIntervalMs: 0 }, + actions: { + run: { + inputSchema: z.unknown(), + block: handler({ + name: "park", + inputSchema: z.unknown(), + outputSchema: z.string(), + execute: async (_input: unknown, ctx) => { + const response = await handleAbortRequest( + new Request(`http://localhost/api/flows/adopt-fence/requests/${requestId}/abort`, { + method: "POST" + }), + { kind: "abort_request", flowKind: "adopt-fence", requestId }, + { stores } + ); + status = response.status; + const aborted = await new Promise((resolve) => { + if (ctx.signal.aborted) return resolve(true); + const timer = setTimeout(() => resolve(false), 300); + ctx.signal.addEventListener("abort", () => { + clearTimeout(timer); + resolve(true); + }, { once: true }); + }); + if (aborted) throw new DOMException("Aborted", "AbortError"); + return "self-completed"; + } + }) + } + } + })({ id: "adopt-fence" }); + + // On file: the later request, which the execution context adopts. + const adopted = createInitialRequestRecord( + { + requestId, + flowKind: "adopt-fence", + flowId: "adopt-fence", + actionName: "run", + userId: "u_adopt", + orgId: DEFAULT_ORG_ID + }, + Date.now() + ); + await stores.request.set(requestId, adopted, "absent"); + + // Admission's read still saw the earlier request under the id. + const get = stores.request.get.bind(stores.request); + let first = true; + stores.request.get = async (id: string) => { + const record = await get(id); + if (first && id === requestId && record !== undefined) { + first = false; + return { ...record, incarnation: "inc_admitted_earlier" }; + } + return record; + }; + + await runAction({ + flow, + actionName: "run", + input: {}, + requestId, + userId: "u_adopt", + orgId: DEFAULT_ORG_ID, + stores, + runtimeConfig: {} + }); + stores.request.get = get; + + expect(status).toBe(204); + const record = await stores.request.get(requestId); + expect(record?.incarnation).toBe(adopted.incarnation); + expect(record?.status).toBe("aborted"); + }); +}); From 2b152c2520d3893cc756c5b063c2685fd8365a90 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 02:38:41 +0000 Subject: [PATCH 04/12] chore(changeset): one user-facing sentence for FIX-1654 Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- .changeset/abort-fence-incarnation-stores.md | 6 ------ .changeset/abort-fence-incarnation.md | 8 +++----- 2 files changed, 3 insertions(+), 11 deletions(-) delete mode 100644 .changeset/abort-fence-incarnation-stores.md diff --git a/.changeset/abort-fence-incarnation-stores.md b/.changeset/abort-fence-incarnation-stores.md deleted file mode 100644 index 0a4201241c..0000000000 --- a/.changeset/abort-fence-incarnation-stores.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@flow-state-dev/store-sqlite": patch -"@flow-state-dev/store-postgres": patch ---- - -`setFieldsIfStatus` fences a cancel on the request's incarnation instead of its `createdAt`, so a cancel reaches the owner's request after their own retry re-stamped it, and never a later request that reused the id (FIX-1654). No schema change; records written before incarnations existed are matched on their `legacy_` value. diff --git a/.changeset/abort-fence-incarnation.md b/.changeset/abort-fence-incarnation.md index 0b15c4c914..61a283521a 100644 --- a/.changeset/abort-fence-incarnation.md +++ b/.changeset/abort-fence-incarnation.md @@ -1,9 +1,7 @@ --- "@flow-state-dev/engine": minor +"@flow-state-dev/store-sqlite": patch +"@flow-state-dev/store-postgres": patch --- -A cancel now lands on the request its caller checked, identified by incarnation rather than `createdAt` (FIX-1654). Before, a cancel issued while the owner's own retry re-stamped the request answered 404 and never stopped the run, and a request created in the same millisecond under a reused id could receive another caller's cancel. - -**Store authors:** `RequestStore.setFieldsIfStatus`'s fifth parameter is now `expectedIncarnation?: string`, replacing `expectedCreatedAt?: number`. Compare it with the stored record's `resolveRequestIncarnation(record)` (newly exported) inside the same atomic step as the status check, and report a mismatch as an absent record. See the engine README, "Abort intent on `RequestStore`". - -The in-process abort registry takes the incarnation too: `registerAbortController(id, incarnation?)`, `abortRequest(id, expectedIncarnation?)` and `hasActiveAbortController(id, expectedIncarnation?)`. Callers that pass no incarnation behave as before. +Cancelling a request now only ever cancels the request whose owner was checked, and `RequestStore.setFieldsIfStatus`'s fifth argument is now that request's expected incarnation, which custom stores must honor (FIX-1654). From d0aaad4c6552bd9526882a41b1f48c57cc33eeea Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 02:50:13 +0000 Subject: [PATCH 05/12] fix(engine): read a stored cancel only for the request this run executes as The catch-up read after a re-tag, the heartbeat intent poll and both abort classifications read the cancel flag by id alone, so a cancel recorded on a later request that took the id could stop or reclassify this run. Each now checks the flag first (the O(1) read) and, only when it is set, reads the record and requires its incarnation to match the one the run executes as. The poll's fire is fenced on it too. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- packages/engine/src/execution/runAction.ts | 43 ++++-- packages/engine/test/abort.test.ts | 144 +++++++++++++++++++++ 2 files changed, 177 insertions(+), 10 deletions(-) diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index 289b3c128c..04d781312d 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -1004,6 +1004,24 @@ export async function runActionInternal< ) )) ); + // The incarnation this run executes as: admission's, until the execution + // context adopts the record and says otherwise (see the re-tag below). + let currentIncarnation = runIncarnation; + /** + * Whether a cancel is recorded on the request this run executes as. The flag + * is read first, as the O(1) `isAbortRequested`, and only when it is set is + * the record read, to check it is still this run's request and not a later + * one that took the id. The common path stays one narrow read. + */ + const abortRecordedForThisRun = async (): Promise => { + if (!(await options.stores.request.isAbortRequested(requestId))) return false; + const record = await options.stores.request.get(requestId); + return ( + record !== undefined && + record.abortRequested === true && + resolveRequestIncarnation(record) === currentIncarnation + ); + }; await registry.register({ requestId, @@ -1068,10 +1086,10 @@ export async function runActionInternal< if (deliveredAbort || abortPollInFlight) return; abortPollInFlight = true; try { - if (!(await options.stores.request.isAbortRequested(requestId))) return; + if (!(await abortRecordedForThisRun())) return; // Returns false when no controller is registered yet. Not a delivery, so // the latch stays unset and the next tick retries. - if (!abortRequest(requestId)) return; + if (!abortRequest(requestId, currentIncarnation)) return; deliveredAbort = true; logRuntimeEvent(logger, "info", "[flow-state] [abort] cross-process abort delivered", { requestId @@ -1637,9 +1655,10 @@ export async function runActionInternal< // one this run executes as, so a cancel fenced on it fires here. A cancel // of that request recorded while the tag was still the earlier one missed // the controller, so read the stored intent once and deliver it. - if (ctx.request.incarnation !== runIncarnation) { - tagAbortController(requestId, abortController, ctx.request.incarnation); - if (await options.stores.request.isAbortRequested(requestId).catch(() => false)) { + if (ctx.request.incarnation !== currentIncarnation) { + currentIncarnation = ctx.request.incarnation; + tagAbortController(requestId, abortController, currentIncarnation); + if (await abortRecordedForThisRun().catch(() => false)) { abortController.abort(); } } @@ -2283,7 +2302,10 @@ export async function runActionInternal< ? await options.stores.request.get(requestId).catch(() => undefined) : undefined; let wasIntentionalAbort = - signalAborted && (deliveredAbort || classificationRecord?.abortRequested === true); + signalAborted && + (deliveredAbort || + (classificationRecord?.abortRequested === true && + resolveRequestIncarnation(classificationRecord) === currentIncarnation)); // The failure path's normalized error is needed for the client-visible // error item, which is emitted before the drain so a caller hears the @@ -2359,10 +2381,11 @@ export async function runActionInternal< // controller, while a stop accepted on another instance only sets the // durable flag and reaches us through the heartbeat's abort poll — // which is running precisely because the heartbeat outlives the drain. - // `isAbortRequested`, not `get()`: this asks one boolean, and the - // interface requires it be O(1) in item count, where `get()` + // `isAbortRequested` first, not `get()`: this asks one boolean, and + // the interface requires it be O(1) in item count, where `get()` // materializes the whole item history to answer it on the persistent - // adapters. A store failure resolves to `false`, which keeps the + // adapters. Only a set flag pays for the `get()` that checks it is + // this run's request. A store failure resolves to `false`, which keeps the // branch already chosen — the same direction the classification read // above fails in. (That read stays on `get()` deliberately: it is the // call that absorbs a transient failure before `patchRequestRecord`, @@ -2370,7 +2393,7 @@ export async function runActionInternal< if (!wasIntentionalAbort) { wasIntentionalAbort = deliveredAbort || - (await options.stores.request.isAbortRequested(requestId).catch(() => false)); + (await abortRecordedForThisRun().catch(() => false)); } } finally { // Clear the heartbeat — the drain is done and the terminal write is diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index e15416886c..4a14db5833 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -2732,4 +2732,148 @@ describe("a run's abort controller carries the incarnation it executes as", () = expect(record?.incarnation).toBe(adopted.incarnation); expect(record?.status).toBe("aborted"); }); + + // The catch-up read after a re-tag answers for whatever holds the id at that + // moment. A cancel recorded on a request that took the id since is not this + // run's, and must not stop it. + it("does not stop the run for a cancel recorded on a request that took the id since", async () => { + const stores = createInMemoryStores(); + const requestId = "req_adopted_then_reused"; + const seen: { aborted?: boolean } = {}; + const flow = defineFlow({ + kind: "adopt-fence-reused", + request: { heartbeatIntervalMs: 0 }, + actions: { + run: { + inputSchema: z.unknown(), + block: handler({ + name: "observe", + inputSchema: z.unknown(), + outputSchema: z.string(), + execute: async (_input: unknown, ctx) => { + seen.aborted = ctx.signal.aborted; + return "ran"; + } + }) + } + } + })({ id: "adopt-fence-reused" }); + + const adopted = createInitialRequestRecord( + { + requestId, + flowKind: "adopt-fence-reused", + flowId: "adopt-fence-reused", + actionName: "run", + userId: "u_adopt", + orgId: DEFAULT_ORG_ID + }, + Date.now() + ); + await stores.request.set(requestId, adopted, "absent"); + + // Admission read an earlier request under the id, so the run re-tags. + const get = stores.request.get.bind(stores.request); + let first = true; + stores.request.get = async (id: string) => { + const record = await get(id); + if (first && id === requestId && record !== undefined) { + first = false; + return { ...record, incarnation: "inc_admitted_earlier" }; + } + return record; + }; + // Before the catch-up read lands, the adopted request's record goes and a + // later request takes the id, and someone cancels that later request. + const isAbortRequested = stores.request.isAbortRequested.bind(stores.request); + let swapped = false; + stores.request.isAbortRequested = async (id: string) => { + if (!swapped && id === requestId) { + swapped = true; + await stores.request.delete(requestId); + await stores.request.set(requestId, { ...adopted, incarnation: "inc_later_request" }, "absent"); + await stores.request.setFieldsIfStatus( + requestId, + { abortRequested: true }, + ["in_progress"], + Date.now(), + "inc_later_request" + ); + } + return isAbortRequested(id); + }; + + await runAction({ + flow, + actionName: "run", + input: {}, + requestId, + userId: "u_adopt", + orgId: DEFAULT_ORG_ID, + stores, + runtimeConfig: {} + }); + stores.request.get = get; + stores.request.isAbortRequested = isAbortRequested; + + expect(swapped).toBe(true); + expect(seen.aborted).toBe(false); + }); + + // The heartbeat's intent poll reads by id too, so it has the same question to + // answer: a cancel recorded on a request that took the id is not this run's. + it("does not let the heartbeat poll deliver a later request's cancel", async () => { + const stores = createInMemoryStores(); + const requestId = "req_poll_reused"; + const seen: { aborted?: boolean } = {}; + const flow = defineFlow({ + kind: "poll-fence", + request: { heartbeatIntervalMs: 10 }, + actions: { + run: { + inputSchema: z.unknown(), + block: handler({ + name: "park", + inputSchema: z.unknown(), + outputSchema: z.string(), + execute: async (_input: unknown, ctx) => { + // The run's record goes and a later request takes the id, and + // someone cancels that later request. + const mine = (await stores.request.get(requestId))!; + await stores.request.delete(requestId); + await stores.request.set( + requestId, + { ...mine, incarnation: "inc_later_request" }, + "absent" + ); + await stores.request.setFieldsIfStatus( + requestId, + { abortRequested: true }, + ["in_progress"], + Date.now(), + "inc_later_request" + ); + // Several poll ticks. + await new Promise((resolve) => setTimeout(resolve, 100)); + seen.aborted = ctx.signal.aborted; + return "ran"; + } + }) + } + } + })({ id: "poll-fence" }); + + await runAction({ + flow, + actionName: "run", + input: {}, + requestId, + userId: "u_poll", + orgId: DEFAULT_ORG_ID, + stores, + runtimeConfig: {} + }); + + expect(seen.aborted).toBe(false); + }); }); From 3cfc7da753f0a4b18ae8a754f44858def1c30058 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 03:04:01 +0000 Subject: [PATCH 06/12] fix(engine): honor a cancel recorded on the request a run adopts With heartbeats off, a run that adopts a record already on file (a queued stub, or a replacement that took the id while the request waited at the gate) never read the stored cancel, and a fenced fire aimed at that request could have missed the earlier controller under the id. The start-of-run intent read now also runs in that case, fenced on the incarnation the run executes as. The BR-12 route test now swaps the controller and the record after the write applies, so a route that decides before the write and fires by id fails it. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- packages/engine/src/execution/runAction.ts | 8 +- packages/engine/test/abort.test.ts | 159 +++++++++++++++++++-- 2 files changed, 152 insertions(+), 15 deletions(-) diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index 04d781312d..de93dff929 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -1453,7 +1453,13 @@ export async function runActionInternal< // controller is deregistered and the delivery has nowhere to land. The cost // is one narrow read on the start path, and `pollAbortIntent` swallows its // own failures, so this can delay a request start but can never fail one. - if (heartbeatTimer !== undefined) await pollAbortIntent(); + // + // With heartbeats off it still runs once when this run adopts a record that + // was already on file (a queued stub, a record a replacement left). A cancel + // may have been recorded on that request before this controller existed, and + // a fire aimed at it could have missed an earlier controller under the id; + // with no ticks to come, this read is the only delivery left. + if (heartbeatTimer !== undefined || admittedRequest !== undefined) await pollAbortIntent(); const composedSignal = options.signal ? AbortSignal.any([options.signal, abortController.signal]) : abortController.signal; diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index 4a14db5833..3b319b7f90 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -397,16 +397,39 @@ describe("handleAbortRequest", () => { }); it("does not fire a later request's controller in this process", async () => { - // The write landed on the request the check read; before the route - // fires locally, that request finished and a later one under the id - // started here. Its controller carries its own incarnation. + // The checked request runs here, so when the route starts, the + // controller under the id is its own. The write lands on it; then, + // before the write's promise resolves and the route fires locally, that + // request finishes and a later one under the id starts here with its own + // controller. Whatever the route decided before the write is stale by + // the time it fires. const checked = record(undefined, 1_000); await stores.request.set(REUSED, checked, "absent"); - const later = registerAbortController(REUSED, "inc_later_request"); + const own = registerAbortController(REUSED, checked.incarnation); + let later: AbortController | undefined; - const status = await abortWithInterleave(undefined, async () => {}); + const write = stores.request.setFieldsIfStatus.bind(stores.request); + stores.request.setFieldsIfStatus = async (...args) => { + const result = await write(...args); + await stores.request.delete(REUSED); + await stores.request.set( + REUSED, + { ...record(undefined, 2_000), incarnation: "inc_later_request" }, + "absent" + ); + later = registerAbortController(REUSED, "inc_later_request"); + return result; + }; + let status: number; + try { + status = await abortWithInterleave(undefined, async () => {}); + } finally { + stores.request.setFieldsIfStatus = write; + } - expect(later.signal.aborted).toBe(false); + expect(own.signal.aborted).toBe(false); + expect(later).toBeDefined(); + expect(later!.signal.aborted).toBe(false); expect(status).toBe(202); }); @@ -2642,6 +2665,114 @@ describe("a queued request cancelled through the route", () => { expect(executed).toEqual([]); expect((await stores.request.get(requestId))?.status).toBe("aborted"); }); + + // While the request waits at the gate, its record can go and a replacement + // by the same owner can take the id. The run adopts whatever holds the id + // when it starts, so a cancel recorded on that replacement is this run's to + // honor, even though the host's queued controller was tagged with the + // earlier incarnation and the route's fenced fire missed it. With heartbeats + // off nothing polls, so the start is the only place left to see it. + it("honors a cancel recorded on the request it adopts at the gate, with heartbeats off", async () => { + const stores = createInMemoryStores(); + const executed: string[] = []; + let releaseHold!: () => void; + const hold = new Promise((resolve) => { + releaseHold = resolve; + }); + let markHolding!: () => void; + const holding = new Promise((resolve) => { + markHolding = resolve; + }); + const concurrency = { policy: "queue", key: "session" } as const; + const registry = createFlowRegistry(); + registry.register( + defineFlow({ + kind: "queued-abort-gate", + request: { heartbeatIntervalMs: 0 }, + actions: { + hold: { + inputSchema: z.object({}), + concurrency, + block: handler({ + name: "hold", + inputSchema: z.object({}), + execute: async () => { + markHolding(); + await hold; + return {}; + } + }) + }, + queued: { + inputSchema: z.object({}), + concurrency, + block: handler({ + name: "queued", + inputSchema: z.object({}), + execute: (_input, ctx) => { + // A handler this short never waits on its signal, so record + // what the run's signal said when it got here. + executed.push(ctx.signal.aborted ? "started cancelled" : "started live"); + return {}; + } + }) + } + } + })({ id: "queued-abort-gate" }) + ); + const host = createInboundTransportHost({ + registry, + stores, + resolvePrincipal: defaultBodyUserIdPrincipalResolver, + runtimeConfig: {} + }); + const dispatch = (action: "hold" | "queued", requestId: string) => + host.dispatch({ + source: "http", + flowKind: "queued-abort-gate", + action, + input: {}, + sessionId: "s_queued_gate", + requestId, + orgId: DEFAULT_ORG_ID, + principal: { userId: "u_queued", orgId: DEFAULT_ORG_ID } + }); + const requestId = "req_queued_gate"; + + const holder = dispatch("hold", "req_queued_gate_holder"); + await holding; + const queued = dispatch("queued", requestId); + for (let i = 0; i < 100 && (await stores.request.get(requestId)) === undefined; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await new Promise((resolve) => setTimeout(resolve, 20)); + + // The queued request's record goes and a replacement by the same owner + // takes the id, then the replacement is cancelled. + const queuedRecord = (await stores.request.get(requestId))!; + await stores.request.delete(requestId); + await stores.request.set( + requestId, + { ...queuedRecord, incarnation: "inc_replacement" }, + "absent" + ); + const response = await handleAbortRequest( + new Request(`http://localhost/api/flows/queued-abort-gate/requests/${requestId}/abort`, { + method: "POST" + }), + { kind: "abort_request", flowKind: "queued-abort-gate", requestId }, + { stores } + ); + expect(response.status).toBe(202); + + releaseHold(); + await holder.finished; + await queued.finished.catch(() => undefined); + + // The run starts with the cancel already delivered, and ends as cancelled. + expect(executed).toEqual(["started cancelled"]); + expect((await stores.request.get(requestId))?.status).toBe("aborted"); + }); }); // The run's controller must carry the incarnation of the request the run @@ -2774,21 +2905,21 @@ describe("a run's abort controller carries the incarnation it executes as", () = // Admission read an earlier request under the id, so the run re-tags. const get = stores.request.get.bind(stores.request); - let first = true; + let reads = 0; stores.request.get = async (id: string) => { const record = await get(id); - if (first && id === requestId && record !== undefined) { - first = false; - return { ...record, incarnation: "inc_admitted_earlier" }; - } + if (id !== requestId || record === undefined) return record; + reads += 1; + if (reads === 1) return { ...record, incarnation: "inc_admitted_earlier" }; return record; }; - // Before the catch-up read lands, the adopted request's record goes and a - // later request takes the id, and someone cancels that later request. + // Before the catch-up read lands (after the context has read and adopted + // the record), the adopted request's record goes and a later request + // takes the id, and someone cancels that later request. const isAbortRequested = stores.request.isAbortRequested.bind(stores.request); let swapped = false; stores.request.isAbortRequested = async (id: string) => { - if (!swapped && id === requestId) { + if (!swapped && id === requestId && reads >= 2) { swapped = true; await stores.request.delete(requestId); await stores.request.set(requestId, { ...adopted, incarnation: "inc_later_request" }, "absent"); From f9e60233b8a76e28bb32663af18dab922ba0f4ec Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 03:15:58 +0000 Subject: [PATCH 07/12] fix(engine): deliver a stored cancel only once the run's request is settled The start-of-run intent read ran before the execution context adopted the record, so a cancel stored on the request admission read could abort the controller (and set the delivered latch) of a different request that took the id before the context's read. The poll now waits until the context has settled which request the run executes as, and the start read runs right after that, against that request only. Heartbeat ticks before then do no reads. Docs now say that with heartbeatIntervalMs: 0 a request recorded before its run started is still checked once at start. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- apps/docs/docs/configuration/flow.md | 2 +- .../docs/docs/server/connection-resilience.md | 2 +- packages/engine/src/execution/runAction.ts | 65 +++++----- packages/engine/src/routes/abort-routes.ts | 3 +- packages/engine/test/abort.test.ts | 122 ++++++++++++++++-- 5 files changed, 152 insertions(+), 42 deletions(-) diff --git a/apps/docs/docs/configuration/flow.md b/apps/docs/docs/configuration/flow.md index e04ba9ce27..dbb7503477 100644 --- a/apps/docs/docs/configuration/flow.md +++ b/apps/docs/docs/configuration/flow.md @@ -168,7 +168,7 @@ The host verifies credentials. The framework applies `defaultUserId` and `requir |-------|------|---------|--------------| | `stateSchema` | Zod object | — | Request-scoped state. | | `onStarted` / `onCompleted` / `onErrored` / `onFinished` / `onStepErrored` | `BlockDefinition` | — | Request lifecycle hooks. | -| `heartbeatIntervalMs` | `number` | `10000` | Active-request heartbeat. `0` disables the heartbeat *and* cross-process abort delivery. | +| `heartbeatIntervalMs` | `number` | `10000` | Active-request heartbeat. `0` disables the heartbeat *and* cross-process abort delivery to a running request. A request recorded before its run started is still checked once for a cancellation when the run starts. | | `sseHeartbeatMs` | `number` | `15000` | SSE `: ping` cadence. `0` disables. | | `concurrency` | `ConcurrencyConfig` | `"allow"` | Default for actions that omit `concurrency`. | | `mutationTimeoutMs` | `number` | `30000` | Budget for in-memory state writes. `Infinity` disables. Scopes that persist — request, session, user, org — are not covered by it. | diff --git a/apps/docs/docs/server/connection-resilience.md b/apps/docs/docs/server/connection-resilience.md index d8282c6722..e5f976e920 100644 --- a/apps/docs/docs/server/connection-resilience.md +++ b/apps/docs/docs/server/connection-resilience.md @@ -129,7 +129,7 @@ The default is 10 seconds. Each tick costs one store read per running request, s ### What your deployment needs 1. **A request store shared across processes** — SQLite or Postgres. The default in-memory store is per-process, so a second process cannot see the record at all. The filesystem store does not qualify either: it assumes a single writer per request and does no inter-process locking. -2. **`heartbeatIntervalMs` greater than 0.** Setting it to `0` disables the timer, and with no tick there is no check and no delivery. The same timer also keeps a running request's registry entry warm, so `0` has a second consequence: a run that lasts longer than the stale-request threshold can be marked `interrupted` while it is still running. See [what a stopped process leaves behind](./background-work.md#what-a-stopped-process-leaves-behind). +2. **`heartbeatIntervalMs` greater than 0.** Setting it to `0` disables the timer, so a cancellation recorded while the request is running is never picked up. One check still happens: a request that was recorded before its run started, such as one waiting in a queue or handed to a worker, is checked once as the run starts, so a cancellation recorded while it waited stops it before it does any work. The same timer also keeps a running request's registry entry warm, so `0` has a second consequence: a run that lasts longer than the stale-request threshold can be marked `interrupted` while it is still running. See [what a stopped process leaves behind](./background-work.md#what-a-stopped-process-leaves-behind). ### What the endpoint returns diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index de93dff929..ee21d97ea4 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -1007,6 +1007,10 @@ export async function runActionInternal< // The incarnation this run executes as: admission's, until the execution // context adopts the record and says otherwise (see the re-tag below). let currentIncarnation = runIncarnation; + // Until the context has adopted the record, which request this run is is not + // settled, so no stored cancel is delivered: one delivered for admission's + // request would stay on this run's controller if another request took the id. + let incarnationSettled = false; /** * Whether a cancel is recorded on the request this run executes as. The flag * is read first, as the O(1) `isAbortRequested`, and only when it is set is @@ -1083,7 +1087,7 @@ export async function runActionInternal< * this closes. */ const pollAbortIntent = async (): Promise => { - if (deliveredAbort || abortPollInFlight) return; + if (deliveredAbort || abortPollInFlight || !incarnationSettled) return; abortPollInFlight = true; try { if (!(await abortRecordedForThisRun())) return; @@ -1437,29 +1441,6 @@ export async function runActionInternal< // If anything between here and the main try block throws, the outer // try/catch below cleans it up. const abortController = registerAbortController(requestId, runIncarnation); - // First poll (FIX-1026). Placed here rather than beside the timer because a - // poll before this line has no controller to fire and therefore cannot - // deliver — it would be a guaranteed-useless store read on every request. - // Here it closes the window where the cancel was recorded between admission - // and the run starting, without waiting a full interval. Gated on the timer - // so `heartbeatIntervalMs: 0` really is off. - // - // AWAITED, not fired and forgotten. The read is issued either way; awaiting - // is what makes "a cancel recorded before the run started stops the run" a - // guarantee instead of a race the store's latency decides. Left unawaited, an - // action shorter than one `isAbortRequested` round trip runs to completion — - // model calls included — clears the post-drain abort check, and persists - // `completed` before the read that would have stopped it returns; by then the - // controller is deregistered and the delivery has nowhere to land. The cost - // is one narrow read on the start path, and `pollAbortIntent` swallows its - // own failures, so this can delay a request start but can never fail one. - // - // With heartbeats off it still runs once when this run adopts a record that - // was already on file (a queued stub, a record a replacement left). A cancel - // may have been recorded on that request before this controller existed, and - // a fire aimed at it could have missed an earlier controller under the id; - // with no ticks to come, this read is the only delivery left. - if (heartbeatTimer !== undefined || admittedRequest !== undefined) await pollAbortIntent(); const composedSignal = options.signal ? AbortSignal.any([options.signal, abortController.signal]) : abortController.signal; @@ -1658,15 +1639,37 @@ export async function runActionInternal< // The controller was tagged from admission's read, but the context reads // the record again and runs as whatever request holds the id now. If // another request took the id in between, re-tag the controller with the - // one this run executes as, so a cancel fenced on it fires here. A cancel - // of that request recorded while the tag was still the earlier one missed - // the controller, so read the stored intent once and deliver it. - if (ctx.request.incarnation !== currentIncarnation) { + // one this run executes as, so a cancel fenced on it fires here. From here + // on, which request this run is is settled. + const adoptedAnother = ctx.request.incarnation !== currentIncarnation; + if (adoptedAnother) { currentIncarnation = ctx.request.incarnation; tagAbortController(requestId, abortController, currentIncarnation); - if (await abortRecordedForThisRun().catch(() => false)) { - abortController.abort(); - } + } + incarnationSettled = true; + + // First poll (FIX-1026), against the request this run executes as. It + // closes the window where the cancel was recorded between admission and + // the run starting, without waiting a full interval: nothing has executed + // yet. Earlier than here, a cancel it read could belong to a request that + // no longer holds the id. + // + // AWAITED, not fired and forgotten. Awaiting is what makes "a cancel + // recorded before the run started stops the run" a guarantee instead of a + // race the store's latency decides. Left unawaited, an action shorter than + // one `isAbortRequested` round trip runs to completion — model calls + // included — clears the post-drain abort check, and persists `completed` + // before the read that would have stopped it returns. The cost is one + // narrow read on the start path, and `pollAbortIntent` swallows its own + // failures, so this can delay a request start but can never fail one. + // + // With heartbeats off it still runs once when this run adopts a record + // that was already on file (a queued stub, a record a replacement left): + // a cancel may have been recorded on that request before this controller + // carried its incarnation, and with no ticks to come this read is the only + // delivery left. Otherwise `heartbeatIntervalMs: 0` does no reads. + if (heartbeatTimer !== undefined || admittedRequest !== undefined || adoptedAnother) { + await pollAbortIntent(); } // Resume mode: load the suspension record + checkpoint to restore the durable diff --git a/packages/engine/src/routes/abort-routes.ts b/packages/engine/src/routes/abort-routes.ts index cd9dec7bb2..de50dc4a2d 100644 --- a/packages/engine/src/routes/abort-routes.ts +++ b/packages/engine/src/routes/abort-routes.ts @@ -27,7 +27,8 @@ type AbortRouteContext = { * a cancel issued anywhere stops a run anywhere. Delivery is bounded by the * flow's `heartbeatIntervalMs` and needs a request store shared across * processes; with `heartbeatIntervalMs: 0` there is no tick and therefore no - * delivery. + * delivery to a running request, only the one read a run makes as it starts + * on a request recorded before it (a queued or dispatched request). * * Returns 204 when the in-memory controller was fired here, 202 when the * intent was recorded for the running process to pick up, 404 if the request diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index 3b319b7f90..6e96d4dd24 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -1307,18 +1307,26 @@ describe("runAction — cross-process abort delivery", () => { const base = createInMemoryStores(); const requestId = "req_xproc_prereg"; const sessionId = "sess_xproc_prereg"; - let pollsBeforeRegistration = 0; + let ticksBeforeRegistration = 0; + let readsBeforeRegistration = 0; // `runAction` installs the heartbeat timer well before it calls // `registerAbortController`, and does session work in between. Making that // session work take real time puts ticks deterministically inside the - // window instead of racing the scheduler for them. + // window instead of racing the scheduler for them. Each tick heartbeats, + // so the heartbeat count shows the ticks landed there. const stores = withStoreOverrides(base, { request: { async isAbortRequested(this: StoreRegistry["request"], id: string) { - if (!hasActiveAbortController(id)) pollsBeforeRegistration += 1; + if (!hasActiveAbortController(id)) readsBeforeRegistration += 1; return Object.getPrototypeOf(this).isAbortRequested.call(this, id); } + }, + activeRequests: { + async heartbeat(this: StoreRegistry["activeRequests"], id: string) { + if (!hasActiveAbortController(id)) ticksBeforeRegistration += 1; + return Object.getPrototypeOf(this).heartbeat.call(this, id); + } } }); const slowSession = Object.create(stores.session) as StoreRegistry["session"]; @@ -1371,7 +1379,10 @@ describe("runAction — cross-process abort delivery", () => { }); // Precondition: ticks really did land in the pre-registration window. - expect(pollsBeforeRegistration).toBeGreaterThan(0); + expect(ticksBeforeRegistration).toBeGreaterThan(0); + // They read nothing there: until the run knows which request it executes + // as, a cancel it read could belong to another request under the id. + expect(readsBeforeRegistration).toBe(0); // And delivery still happened. Under a DETECTION latch those first ticks // would have latched on a read they could not act on, suppressed every // later poll, and let the run finish naturally. @@ -2864,9 +2875,9 @@ describe("a run's abort controller carries the incarnation it executes as", () = expect(record?.status).toBe("aborted"); }); - // The catch-up read after a re-tag answers for whatever holds the id at that - // moment. A cancel recorded on a request that took the id since is not this - // run's, and must not stop it. + // The stored-cancel read at the start of the run answers for whatever holds + // the id at that moment. A cancel recorded on a request that took the id + // since the context adopted the record is not this run's, and must not stop it. it("does not stop the run for a cancel recorded on a request that took the id since", async () => { const stores = createInMemoryStores(); const requestId = "req_adopted_then_reused"; @@ -2913,7 +2924,7 @@ describe("a run's abort controller carries the incarnation it executes as", () = if (reads === 1) return { ...record, incarnation: "inc_admitted_earlier" }; return record; }; - // Before the catch-up read lands (after the context has read and adopted + // Before the start read lands (after the context has read and adopted // the record), the adopted request's record goes and a later request // takes the id, and someone cancels that later request. const isAbortRequested = stores.request.isAbortRequested.bind(stores.request); @@ -3007,4 +3018,99 @@ describe("a run's abort controller carries the incarnation it executes as", () = expect(seen.aborted).toBe(false); }); + + // A cancel recorded on the request admission read belongs to that request. + // If another request takes the id before the context reads the record, the + // run executes as that other request, which nobody cancelled. + it("does not start the adopted request cancelled by the earlier request's stored cancel", async () => { + const stores = createInMemoryStores(); + const requestId = "req_cancelled_then_replaced"; + const seen: { aborted?: boolean } = {}; + const flow = defineFlow({ + kind: "adopt-fence-cancelled", + request: { heartbeatIntervalMs: 0 }, + actions: { + run: { + inputSchema: z.unknown(), + block: handler({ + name: "observe", + inputSchema: z.unknown(), + outputSchema: z.string(), + execute: async (_input: unknown, ctx) => { + seen.aborted = ctx.signal.aborted; + return "ran"; + } + }) + } + } + })({ id: "adopt-fence-cancelled" }); + const fresh = () => + createInitialRequestRecord( + { + requestId, + flowKind: "adopt-fence-cancelled", + flowId: "adopt-fence-cancelled", + actionName: "run", + userId: "u_adopt", + orgId: DEFAULT_ORG_ID + }, + Date.now() + ); + + // On file: the earlier request, already cancelled. + const earlier = fresh(); + await stores.request.set(requestId, earlier, "absent"); + await stores.request.setFieldsIfStatus( + requestId, + { abortRequested: true }, + ["in_progress"], + Date.now(), + earlier.incarnation + ); + + // Admission reads the earlier request. The read that decides which + // request the run executes as (any later read not made to confirm a + // stored cancel) finds a replacement that took the id. + const later = fresh(); + const get = stores.request.get.bind(stores.request); + const isAbortRequested = stores.request.isAbortRequested.bind(stores.request); + let reads = 0; + let confirming = false; + let swapped = false; + stores.request.isAbortRequested = async (id: string) => { + confirming = true; + return isAbortRequested(id); + }; + stores.request.get = async (id: string) => { + if (id === requestId) { + reads += 1; + if (reads > 1 && !confirming && !swapped) { + swapped = true; + await stores.request.delete(requestId); + await stores.request.set(requestId, later, "absent"); + } + confirming = false; + } + return get(id); + }; + + await runAction({ + flow, + actionName: "run", + input: {}, + requestId, + userId: "u_adopt", + orgId: DEFAULT_ORG_ID, + stores, + runtimeConfig: {} + }); + stores.request.get = get; + stores.request.isAbortRequested = isAbortRequested; + + const record = await stores.request.get(requestId); + expect(swapped).toBe(true); + expect(record?.incarnation).toBe(later.incarnation); + expect(seen.aborted).toBe(false); + expect(record?.status).toBe("completed"); + }); }); From 73c34a4b0abafb81e019b566e145fbfd1b5cebec Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 03:31:52 +0000 Subject: [PATCH 08/12] fix(engine): every run checks for a stored cancel once it knows its request The start read ran only under a condition that three review rounds found gaps in; the latest was a claim that takes over a record already cancelled, with heartbeats off. It now runs for every run, once the context has settled which request the run executes as: one O(1) read before any block runs. The run's signal is now its own controller, fed by the registered one only after that settle point. A fire that landed earlier, for a request that then lost the id to the one the context adopted, is dropped with its controller instead of starting the adopted request cancelled. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- apps/docs/docs/configuration/flow.md | 2 +- .../docs/docs/server/connection-resilience.md | 2 +- packages/engine/src/execution/runAction.ts | 47 +++-- packages/engine/src/routes/abort-routes.ts | 4 +- packages/engine/test/abort.test.ts | 169 ++++++++++++++++++ 5 files changed, 205 insertions(+), 19 deletions(-) diff --git a/apps/docs/docs/configuration/flow.md b/apps/docs/docs/configuration/flow.md index dbb7503477..5627bd02fe 100644 --- a/apps/docs/docs/configuration/flow.md +++ b/apps/docs/docs/configuration/flow.md @@ -168,7 +168,7 @@ The host verifies credentials. The framework applies `defaultUserId` and `requir |-------|------|---------|--------------| | `stateSchema` | Zod object | — | Request-scoped state. | | `onStarted` / `onCompleted` / `onErrored` / `onFinished` / `onStepErrored` | `BlockDefinition` | — | Request lifecycle hooks. | -| `heartbeatIntervalMs` | `number` | `10000` | Active-request heartbeat. `0` disables the heartbeat *and* cross-process abort delivery to a running request. A request recorded before its run started is still checked once for a cancellation when the run starts. | +| `heartbeatIntervalMs` | `number` | `10000` | Active-request heartbeat. `0` disables the heartbeat *and* cross-process abort delivery to a running request. Each run still checks once for a cancellation as it starts. | | `sseHeartbeatMs` | `number` | `15000` | SSE `: ping` cadence. `0` disables. | | `concurrency` | `ConcurrencyConfig` | `"allow"` | Default for actions that omit `concurrency`. | | `mutationTimeoutMs` | `number` | `30000` | Budget for in-memory state writes. `Infinity` disables. Scopes that persist — request, session, user, org — are not covered by it. | diff --git a/apps/docs/docs/server/connection-resilience.md b/apps/docs/docs/server/connection-resilience.md index e5f976e920..ae27c821c9 100644 --- a/apps/docs/docs/server/connection-resilience.md +++ b/apps/docs/docs/server/connection-resilience.md @@ -129,7 +129,7 @@ The default is 10 seconds. Each tick costs one store read per running request, s ### What your deployment needs 1. **A request store shared across processes** — SQLite or Postgres. The default in-memory store is per-process, so a second process cannot see the record at all. The filesystem store does not qualify either: it assumes a single writer per request and does no inter-process locking. -2. **`heartbeatIntervalMs` greater than 0.** Setting it to `0` disables the timer, so a cancellation recorded while the request is running is never picked up. One check still happens: a request that was recorded before its run started, such as one waiting in a queue or handed to a worker, is checked once as the run starts, so a cancellation recorded while it waited stops it before it does any work. The same timer also keeps a running request's registry entry warm, so `0` has a second consequence: a run that lasts longer than the stale-request threshold can be marked `interrupted` while it is still running. See [what a stopped process leaves behind](./background-work.md#what-a-stopped-process-leaves-behind). +2. **`heartbeatIntervalMs` greater than 0.** Setting it to `0` disables the timer, so a cancellation recorded while the request is running is never picked up. One check still happens: each run checks once as it starts, so a cancellation recorded before then, such as while the request waited in a queue, stops it before it does any work. The same timer also keeps a running request's registry entry warm, so `0` has a second consequence: a run that lasts longer than the stale-request threshold can be marked `interrupted` while it is still running. See [what a stopped process leaves behind](./background-work.md#what-a-stopped-process-leaves-behind). ### What the endpoint returns diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index ee21d97ea4..e6e6f22b2a 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -1440,7 +1440,19 @@ export async function runActionInternal< // If anything above threw, the controller would never be registered. // If anything between here and the main try block throws, the outer // try/catch below cleans it up. - const abortController = registerAbortController(requestId, runIncarnation); + // + // Two controllers. `registered` is the one in the registry, which the abort + // endpoint and the poll fire. `abortController` is this run's own signal. + // A fire reaches the run only once the context has settled which request the + // run executes as: one that landed earlier, for a request that then lost the + // id to the one the context adopted, belongs to that other request and is + // dropped with its controller (see the settle step below). + let registered = registerAbortController(requestId, runIncarnation); + const abortController = new AbortController(); + const forwardRegisteredAbort = (): void => { + if (incarnationSettled) abortController.abort(registered.signal.reason); + }; + registered.signal.addEventListener("abort", forwardRegisteredAbort, { once: true }); const composedSignal = options.signal ? AbortSignal.any([options.signal, abortController.signal]) : abortController.signal; @@ -1638,15 +1650,22 @@ export async function runActionInternal< // The controller was tagged from admission's read, but the context reads // the record again and runs as whatever request holds the id now. If - // another request took the id in between, re-tag the controller with the - // one this run executes as, so a cancel fenced on it fires here. From here - // on, which request this run is is settled. - const adoptedAnother = ctx.request.incarnation !== currentIncarnation; - if (adoptedAnother) { + // another request took the id in between, the registered controller must + // answer to the one this run executes as. Re-tag it, or, if a fire for the + // earlier request already landed on it, replace it: that fire was not for + // this request. From here on, which request this run is is settled, and a + // fire that landed for it before now reaches the run. + if (ctx.request.incarnation !== currentIncarnation) { currentIncarnation = ctx.request.incarnation; - tagAbortController(requestId, abortController, currentIncarnation); + if (registered.signal.aborted) { + registered = registerAbortController(requestId, currentIncarnation); + registered.signal.addEventListener("abort", forwardRegisteredAbort, { once: true }); + } else { + tagAbortController(requestId, registered, currentIncarnation); + } } incarnationSettled = true; + if (registered.signal.aborted) forwardRegisteredAbort(); // First poll (FIX-1026), against the request this run executes as. It // closes the window where the cancel was recorded between admission and @@ -1663,14 +1682,12 @@ export async function runActionInternal< // narrow read on the start path, and `pollAbortIntent` swallows its own // failures, so this can delay a request start but can never fail one. // - // With heartbeats off it still runs once when this run adopts a record - // that was already on file (a queued stub, a record a replacement left): - // a cancel may have been recorded on that request before this controller - // carried its incarnation, and with no ticks to come this read is the only - // delivery left. Otherwise `heartbeatIntervalMs: 0` does no reads. - if (heartbeatTimer !== undefined || admittedRequest !== undefined || adoptedAnother) { - await pollAbortIntent(); - } + // Every run makes it, heartbeats on or off: a cancel can be recorded on + // this request before its controller carried the incarnation (a queued + // stub, a record a replacement left, a record a claim took over), and with + // heartbeats off this read is the only delivery there is. It is one O(1) + // `isAbortRequested` read per run. + await pollAbortIntent(); // Resume mode: load the suspension record + checkpoint to restore the durable // sequencer's accumulator state. `resumeOf` (legacy two-request path) reads diff --git a/packages/engine/src/routes/abort-routes.ts b/packages/engine/src/routes/abort-routes.ts index de50dc4a2d..45c707fa8f 100644 --- a/packages/engine/src/routes/abort-routes.ts +++ b/packages/engine/src/routes/abort-routes.ts @@ -27,8 +27,8 @@ type AbortRouteContext = { * a cancel issued anywhere stops a run anywhere. Delivery is bounded by the * flow's `heartbeatIntervalMs` and needs a request store shared across * processes; with `heartbeatIntervalMs: 0` there is no tick and therefore no - * delivery to a running request, only the one read a run makes as it starts - * on a request recorded before it (a queued or dispatched request). + * delivery to a running request, only the one check each run makes as it + * starts. * * Returns 204 when the in-memory controller was fired here, 202 when the * intent was recorded for the running process to pick up, 404 if the request diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index 6e96d4dd24..d6400b157f 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -3113,4 +3113,173 @@ describe("a run's abort controller carries the incarnation it executes as", () = expect(seen.aborted).toBe(false); expect(record?.status).toBe("completed"); }); + + // Admission found no record, and the owner's other run created the request + // and had it cancelled before this run's claim. The claim hands the record + // off, keeping its incarnation and its recorded cancel: this run is that + // request, so it starts cancelled. Heartbeats are off, so the read at start + // is the only delivery. + it("honors a cancel recorded on the record its claim takes over, with heartbeats off", async () => { + const stores = createInMemoryStores(); + const requestId = "req_claimed_cancelled"; + const seen: { aborted?: boolean } = {}; + const flow = defineFlow({ + kind: "claim-cancelled", + request: { heartbeatIntervalMs: 0 }, + actions: { + run: { + inputSchema: z.unknown(), + block: handler({ + name: "observe", + inputSchema: z.unknown(), + outputSchema: z.string(), + execute: async (_input: unknown, ctx) => { + seen.aborted = ctx.signal.aborted; + return "ran"; + } + }) + } + } + })({ id: "claim-cancelled" }); + + const holder = createInitialRequestRecord( + { + requestId, + flowKind: "claim-cancelled", + flowId: "claim-cancelled", + actionName: "run", + userId: "u_claim", + orgId: DEFAULT_ORG_ID + }, + Date.now() + ); + const set = stores.request.set.bind(stores.request); + let raced = false; + stores.request.set = async (id, record, expected) => { + if (!raced && id === requestId && expected === "absent") { + raced = true; + await set(requestId, holder, "absent"); + await stores.request.setFieldsIfStatus( + requestId, + { abortRequested: true }, + ["in_progress"], + Date.now(), + holder.incarnation + ); + } + return set(id, record, expected); + }; + + await runAction({ + flow, + actionName: "run", + input: {}, + requestId, + userId: "u_claim", + orgId: DEFAULT_ORG_ID, + stores, + runtimeConfig: {} + }); + stores.request.set = set; + + const record = await stores.request.get(requestId); + expect(raced).toBe(true); + expect(record?.incarnation).toBe(holder.incarnation); + expect(seen.aborted).toBe(true); + expect(record?.status).toBe("aborted"); + }); + + // A cancel fired here for the request admission read can land while the + // context is being built. If another request then takes the id before the + // context reads the record, the run executes as that other request, which + // nobody cancelled: the fire must not carry over to it. + it("does not carry a fire aimed at the earlier request over to the adopted one", async () => { + const stores = createInMemoryStores(); + const requestId = "req_fired_then_replaced"; + const seen: { aborted?: boolean } = {}; + let status: number | undefined; + const flow = defineFlow({ + kind: "fire-then-replace", + request: { heartbeatIntervalMs: 0 }, + actions: { + run: { + inputSchema: z.unknown(), + block: handler({ + name: "observe", + inputSchema: z.unknown(), + outputSchema: z.string(), + execute: async (_input: unknown, ctx) => { + seen.aborted = ctx.signal.aborted; + return "ran"; + } + }) + } + } + })({ id: "fire-then-replace" }); + const fresh = () => + createInitialRequestRecord( + { + requestId, + flowKind: "fire-then-replace", + flowId: "fire-then-replace", + actionName: "run", + userId: "u_fire", + orgId: DEFAULT_ORG_ID + }, + Date.now() + ); + + const earlier = fresh(); + await stores.request.set(requestId, earlier, "absent"); + const later = fresh(); + + // Admission's read is the first; the context's is the second. Just before + // the context reads, the earlier request is cancelled through the route + // (fired here), then replaced by the later one. + const get = stores.request.get.bind(stores.request); + let reads = 0; + let inRoute = false; + stores.request.get = async (id: string) => { + if (id === requestId && !inRoute) { + reads += 1; + if (reads === 2) { + inRoute = true; + try { + const response = await handleAbortRequest( + new Request(`http://localhost/api/flows/fire-then-replace/requests/${requestId}/abort`, { + method: "POST" + }), + { kind: "abort_request", flowKind: "fire-then-replace", requestId }, + { stores } + ); + status = response.status; + } finally { + inRoute = false; + } + await stores.request.delete(requestId); + await stores.request.set(requestId, later, "absent"); + } + } + return get(id); + }; + + await runAction({ + flow, + actionName: "run", + input: {}, + requestId, + userId: "u_fire", + orgId: DEFAULT_ORG_ID, + stores, + runtimeConfig: {} + }); + stores.request.get = get; + + const record = await stores.request.get(requestId); + // Precondition: the fire really landed on this run's controller. + expect(status).toBe(204); + expect(record?.incarnation).toBe(later.incarnation); + expect(seen.aborted).toBe(false); + expect(record?.status).toBe("completed"); + }); }); From b9e0d8a82c03d484d2fbb55499b17366074b5e80 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 03:44:22 +0000 Subject: [PATCH 09/12] fix(engine): keep an unfenced abort across the settle-time controller swap Round 7 replaced the registered controller whenever it had fired before the context adopted another request. An unfenced fire (shutdown cancelling a detached child, the CLI stopping its own turn) stores nothing, so the start read cannot redeliver it, and the swap threw it away. The registry now records whether a fire was fenced, and the swap drops only a fire fenced on the earlier incarnation. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- .../engine/src/execution/abort-registry.ts | 18 +++++ packages/engine/src/execution/runAction.ts | 15 ++-- packages/engine/test/abort.test.ts | 81 +++++++++++++++++++ 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/packages/engine/src/execution/abort-registry.ts b/packages/engine/src/execution/abort-registry.ts index 020b18ab5b..fc30b65373 100644 --- a/packages/engine/src/execution/abort-registry.ts +++ b/packages/engine/src/execution/abort-registry.ts @@ -23,6 +23,12 @@ interface Registered { controller: AbortController; /** The incarnation of the request this controller belongs to, once known. */ incarnation?: string; + /** + * Set when the controller was fired by a fenced `abortRequest`, so the fire + * was for the request the controller was tagged with at that moment. Unset + * for an unfenced fire, which is for whatever runs under the id. + */ + firedFenced?: boolean; } const controllers = new Map(); @@ -76,10 +82,22 @@ export function abortRequest(requestId: string, expectedIncarnation?: string): b if (entry === undefined || !matches(entry, expectedIncarnation)) { return false; } + if (!entry.controller.signal.aborted) entry.firedFenced = expectedIncarnation !== undefined; entry.controller.abort(); return true; } +/** + * Whether `controller`, registered under `requestId`, was fired by a fenced + * `abortRequest`: a fire for one incarnation of the request, not for whatever + * runs under the id. False when it has not fired, was fired unfenced, or is no + * longer the registered controller. + */ +export function wasFiredFenced(requestId: string, controller: AbortController): boolean { + const entry = controllers.get(requestId); + return entry !== undefined && entry.controller === controller && entry.firedFenced === true; +} + /** * Remove the controller from the registry. Called on any terminal state. */ diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index e6e6f22b2a..ce293fa839 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -62,7 +62,8 @@ import { abortRequest, registerAbortController, deregisterAbortController, - tagAbortController + tagAbortController, + wasFiredFenced } from "./abort-registry"; import { assertSessionAdmitted, @@ -1651,13 +1652,15 @@ export async function runActionInternal< // The controller was tagged from admission's read, but the context reads // the record again and runs as whatever request holds the id now. If // another request took the id in between, the registered controller must - // answer to the one this run executes as. Re-tag it, or, if a fire for the - // earlier request already landed on it, replace it: that fire was not for - // this request. From here on, which request this run is is settled, and a - // fire that landed for it before now reaches the run. + // answer to the one this run executes as. Re-tag it, or, if a fire fenced + // on the earlier request already landed on it, replace it: that fire was + // not for this request. An unfenced fire (shutdown, a CLI stopping its own + // turn) was for whatever runs under the id, so it stays. From here on, + // which request this run is is settled, and a fire that stays reaches the + // run. if (ctx.request.incarnation !== currentIncarnation) { currentIncarnation = ctx.request.incarnation; - if (registered.signal.aborted) { + if (wasFiredFenced(requestId, registered)) { registered = registerAbortController(requestId, currentIncarnation); registered.signal.addEventListener("abort", forwardRegisteredAbort, { once: true }); } else { diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index d6400b157f..ab7dd32bf9 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -3282,4 +3282,85 @@ describe("a run's abort controller carries the incarnation it executes as", () = expect(seen.aborted).toBe(false); expect(record?.status).toBe("completed"); }); + + // A fire with no incarnation (shutdown cancelling a detached child, a CLI + // stopping its own turn) means "stop whatever runs under this id". Nothing + // is stored for it, so the start read cannot redeliver it: it has to survive + // the context adopting another request. + it("carries an unfenced fire over to the adopted request", async () => { + const stores = createInMemoryStores(); + const requestId = "req_unfenced_then_replaced"; + const seen: { aborted?: boolean } = {}; + let fired: boolean | undefined; + const flow = defineFlow({ + kind: "unfenced-then-replace", + request: { heartbeatIntervalMs: 0 }, + actions: { + run: { + inputSchema: z.unknown(), + block: handler({ + name: "observe", + inputSchema: z.unknown(), + outputSchema: z.string(), + execute: async (_input: unknown, ctx) => { + seen.aborted = ctx.signal.aborted; + if (ctx.signal.aborted) throw new DOMException("Aborted", "AbortError"); + return "ran"; + } + }) + } + } + })({ id: "unfenced-then-replace" }); + const fresh = () => + createInitialRequestRecord( + { + requestId, + flowKind: "unfenced-then-replace", + flowId: "unfenced-then-replace", + actionName: "run", + userId: "u_unfenced", + orgId: DEFAULT_ORG_ID + }, + Date.now() + ); + + await stores.request.set(requestId, fresh(), "absent"); + const later = fresh(); + + // Admission's read is the first; the context's is the second. Just before + // the context reads, an unfenced fire lands, then another request takes + // the id. + const get = stores.request.get.bind(stores.request); + let reads = 0; + stores.request.get = async (id: string) => { + if (id === requestId) { + reads += 1; + if (reads === 2) { + fired = abortRequest(requestId); + await stores.request.delete(requestId); + await stores.request.set(requestId, later, "absent"); + } + } + return get(id); + }; + + await runAction({ + flow, + actionName: "run", + input: {}, + requestId, + userId: "u_unfenced", + orgId: DEFAULT_ORG_ID, + stores, + runtimeConfig: {} + }); + stores.request.get = get; + + const record = await stores.request.get(requestId); + // Precondition: the fire landed on this run's controller. + expect(fired).toBe(true); + expect(record?.incarnation).toBe(later.incarnation); + expect(seen.aborted).toBe(true); + expect(record?.status).not.toBe("completed"); + }); }); From 35f52685b98c1ca074d4fc55b86f904f901d22bc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 03:55:01 +0000 Subject: [PATCH 10/12] fix(engine): an unfenced abort survives the swap in any order An unfenced fire that landed after a fenced one recorded nothing, since the controller had already fired, so the settle-time swap still dropped it. The registry now records fenced and unfenced fires separately, the unfenced one even on an already-fired controller, and the swap happens only when every pre-settle fire was fenced. A table test covers every pre-settle fire order against adopting the same or another request. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- .../engine/src/execution/abort-registry.ts | 31 ++++-- packages/engine/src/execution/runAction.ts | 10 +- packages/engine/test/abort.test.ts | 103 ++++++++++++++++++ 3 files changed, 129 insertions(+), 15 deletions(-) diff --git a/packages/engine/src/execution/abort-registry.ts b/packages/engine/src/execution/abort-registry.ts index fc30b65373..c37ceb73e3 100644 --- a/packages/engine/src/execution/abort-registry.ts +++ b/packages/engine/src/execution/abort-registry.ts @@ -24,11 +24,16 @@ interface Registered { /** The incarnation of the request this controller belongs to, once known. */ incarnation?: string; /** - * Set when the controller was fired by a fenced `abortRequest`, so the fire - * was for the request the controller was tagged with at that moment. Unset - * for an unfenced fire, which is for whatever runs under the id. + * Set by a fenced `abortRequest` that matched: a fire for the request the + * controller was tagged with at that moment. */ firedFenced?: boolean; + /** + * Set by an unfenced `abortRequest`: a fire for whatever runs under the id. + * Recorded even when the controller had already fired, so it is never + * hidden behind an earlier fenced fire. + */ + firedUnfenced?: boolean; } const controllers = new Map(); @@ -82,20 +87,26 @@ export function abortRequest(requestId: string, expectedIncarnation?: string): b if (entry === undefined || !matches(entry, expectedIncarnation)) { return false; } - if (!entry.controller.signal.aborted) entry.firedFenced = expectedIncarnation !== undefined; + if (expectedIncarnation === undefined) entry.firedUnfenced = true; + else entry.firedFenced = true; entry.controller.abort(); return true; } /** - * Whether `controller`, registered under `requestId`, was fired by a fenced - * `abortRequest`: a fire for one incarnation of the request, not for whatever - * runs under the id. False when it has not fired, was fired unfenced, or is no - * longer the registered controller. + * Whether `controller`, registered under `requestId`, was fired only by fenced + * `abortRequest`s: fires for one incarnation of the request, and none for + * whatever runs under the id. False when it has not fired, when any fire was + * unfenced, or when it is no longer the registered controller. */ -export function wasFiredFenced(requestId: string, controller: AbortController): boolean { +export function wasFiredOnlyFenced(requestId: string, controller: AbortController): boolean { const entry = controllers.get(requestId); - return entry !== undefined && entry.controller === controller && entry.firedFenced === true; + return ( + entry !== undefined && + entry.controller === controller && + entry.firedFenced === true && + entry.firedUnfenced !== true + ); } /** diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index ce293fa839..c41ff16ae8 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -63,7 +63,7 @@ import { registerAbortController, deregisterAbortController, tagAbortController, - wasFiredFenced + wasFiredOnlyFenced } from "./abort-registry"; import { assertSessionAdmitted, @@ -1652,15 +1652,15 @@ export async function runActionInternal< // The controller was tagged from admission's read, but the context reads // the record again and runs as whatever request holds the id now. If // another request took the id in between, the registered controller must - // answer to the one this run executes as. Re-tag it, or, if a fire fenced - // on the earlier request already landed on it, replace it: that fire was - // not for this request. An unfenced fire (shutdown, a CLI stopping its own + // answer to the one this run executes as. Re-tag it, or, if only fires + // fenced on the earlier request landed on it, replace it: those were not + // for this request. Any unfenced fire (shutdown, a CLI stopping its own // turn) was for whatever runs under the id, so it stays. From here on, // which request this run is is settled, and a fire that stays reaches the // run. if (ctx.request.incarnation !== currentIncarnation) { currentIncarnation = ctx.request.incarnation; - if (wasFiredFenced(requestId, registered)) { + if (wasFiredOnlyFenced(requestId, registered)) { registered = registerAbortController(requestId, currentIncarnation); registered.signal.addEventListener("abort", forwardRegisteredAbort, { once: true }); } else { diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index ab7dd32bf9..542314cb08 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -3363,4 +3363,107 @@ describe("a run's abort controller carries the incarnation it executes as", () = expect(seen.aborted).toBe(true); expect(record?.status).not.toBe("completed"); }); + + // Every order of fires that can land on the run's controller before the + // context settles which request the run is, crossed with whether the + // context adopts the request admission read or another that took the id. + // The run starts cancelled exactly when an unfenced fire landed (it was for + // whatever runs under the id) or a fenced fire targeted the request the run + // turned out to be. Before settling, a fenced fire can only target the + // request admission read, so on the other request only an unfenced fire + // counts. + describe("pre-settle fires, by order and by the request adopted", () => { + type Fire = "fenced" | "unfenced"; + const sequences: Array<[string, Fire[]]> = [ + ["no fire", []], + ["a fenced fire", ["fenced"]], + ["an unfenced fire", ["unfenced"]], + ["a fenced then an unfenced fire", ["fenced", "unfenced"]], + ["an unfenced then a fenced fire", ["unfenced", "fenced"]] + ]; + const cases = sequences.flatMap(([label, fires]) => [ + { label, fires, adopts: "same" as const, startsAborted: fires.length > 0 }, + { label, fires, adopts: "another" as const, startsAborted: fires.includes("unfenced") } + ]); + + it.each(cases)( + "$label, adopting the $adopts request: starts aborted = $startsAborted", + async ({ fires, adopts, startsAborted }) => { + const stores = createInMemoryStores(); + const requestId = `req_presettle_${fires.join("_") || "none"}_${adopts}`; + const seen: { aborted?: boolean } = {}; + const kind = "presettle-fires"; + const flow = defineFlow({ + kind, + request: { heartbeatIntervalMs: 0 }, + actions: { + run: { + inputSchema: z.unknown(), + block: handler({ + name: "observe", + inputSchema: z.unknown(), + outputSchema: z.string(), + execute: async (_input: unknown, ctx) => { + seen.aborted = ctx.signal.aborted; + if (ctx.signal.aborted) throw new DOMException("Aborted", "AbortError"); + return "ran"; + } + }) + } + } + })({ id: kind }); + const fresh = () => + createInitialRequestRecord( + { requestId, flowKind: kind, flowId: kind, actionName: "run", userId: "u_presettle", orgId: DEFAULT_ORG_ID }, + Date.now() + ); + + const admitted = fresh(); + await stores.request.set(requestId, admitted, "absent"); + const other = fresh(); + + // Admission's read is the first; the context's is the second. The + // fires land just before the context reads. + const get = stores.request.get.bind(stores.request); + let reads = 0; + const landed: boolean[] = []; + stores.request.get = async (id: string) => { + if (id === requestId) { + reads += 1; + if (reads === 2) { + for (const fire of fires) { + landed.push( + fire === "fenced" ? abortRequest(requestId, admitted.incarnation) : abortRequest(requestId) + ); + } + if (adopts === "another") { + await stores.request.delete(requestId); + await stores.request.set(requestId, other, "absent"); + } + } + } + return get(id); + }; + + await runAction({ + flow, + actionName: "run", + input: {}, + requestId, + userId: "u_presettle", + orgId: DEFAULT_ORG_ID, + stores, + runtimeConfig: {} + }); + stores.request.get = get; + + // Precondition: every fire reached this run's controller. + expect(landed).toEqual(fires.map(() => true)); + expect((await stores.request.get(requestId))?.incarnation).toBe( + adopts === "same" ? admitted.incarnation : other.incarnation + ); + expect(seen.aborted).toBe(startsAborted); + } + ); + }); }); From 8cc7ca6110dcfed8f143bf4893dfc412a6e3f4d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 04:10:27 +0000 Subject: [PATCH 11/12] fix(engine): keep abort provenance on the controller; fence the queued gate's terminal writes (FIX-1654) How a controller was fired is now recorded on the controller itself, so a run whose controller another run displaced still knows whether a pre-settle fire was fenced on the earlier request or unfenced. A fired controller is replaced at settle only if it still holds the registry slot. The host's queued path ends only the request it claimed: a cancel for A no longer terminates B when B has taken the id before the gate opens. The run adopts B on a fresh controller and B's start read settles B's own cancel. The queue-timeout and external-dispatch failure writes are fenced the same way. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- .../engine/src/execution/abort-registry.ts | 61 ++++--- packages/engine/src/execution/runAction.ts | 5 +- .../host/createInboundTransportHost.ts | 78 +++++++-- packages/engine/test/abort.test.ts | 164 ++++++++++++++++-- 4 files changed, 251 insertions(+), 57 deletions(-) diff --git a/packages/engine/src/execution/abort-registry.ts b/packages/engine/src/execution/abort-registry.ts index c37ceb73e3..bd184fc7ad 100644 --- a/packages/engine/src/execution/abort-registry.ts +++ b/packages/engine/src/execution/abort-registry.ts @@ -23,21 +23,19 @@ interface Registered { controller: AbortController; /** The incarnation of the request this controller belongs to, once known. */ incarnation?: string; - /** - * Set by a fenced `abortRequest` that matched: a fire for the request the - * controller was tagged with at that moment. - */ - firedFenced?: boolean; - /** - * Set by an unfenced `abortRequest`: a fire for whatever runs under the id. - * Recorded even when the controller had already fired, so it is never - * hidden behind an earlier fenced fire. - */ - firedUnfenced?: boolean; } const controllers = new Map(); +/** + * How each controller was fired, kept on the controller rather than on its + * registry slot, so the answer survives another run taking the slot. + * `fenced`: a fire for the incarnation the controller was tagged with then. + * `unfenced`: a fire for whatever runs under the id; recorded even when the + * controller had already fired, so an earlier fenced fire never hides it. + */ +const firedBy = new WeakMap(); + /** True when `entry` may be acted on by a caller expecting `expected`. */ function matches(entry: Registered, expected: string | undefined): boolean { return expected === undefined || entry.incarnation === expected; @@ -87,26 +85,39 @@ export function abortRequest(requestId: string, expectedIncarnation?: string): b if (entry === undefined || !matches(entry, expectedIncarnation)) { return false; } - if (expectedIncarnation === undefined) entry.firedUnfenced = true; - else entry.firedFenced = true; + const fired = firedBy.get(entry.controller) ?? {}; + if (expectedIncarnation === undefined) fired.unfenced = true; + else fired.fenced = true; + firedBy.set(entry.controller, fired); entry.controller.abort(); return true; } /** - * Whether `controller`, registered under `requestId`, was fired only by fenced - * `abortRequest`s: fires for one incarnation of the request, and none for - * whatever runs under the id. False when it has not fired, when any fire was - * unfenced, or when it is no longer the registered controller. + * Whether `controller` was fired only by fenced `abortRequest`s: fires for one + * incarnation of its request, and none for whatever runs under the id. False + * when it has not fired or when any fire was unfenced. Answered whether or not + * the controller is still registered. */ -export function wasFiredOnlyFenced(requestId: string, controller: AbortController): boolean { - const entry = controllers.get(requestId); - return ( - entry !== undefined && - entry.controller === controller && - entry.firedFenced === true && - entry.firedUnfenced !== true - ); +export function wasFiredOnlyFenced(controller: AbortController): boolean { + const fired = firedBy.get(controller); + return fired?.fenced === true && fired.unfenced !== true; +} + +/** + * Give up `previous` for a fresh, unfired controller tagged `incarnation`. + * The fresh one takes the registry slot only if `previous` still holds it; + * a controller another run displaced does not take the slot back. + */ +export function replaceAbortController( + requestId: string, + previous: AbortController, + incarnation: string +): AbortController { + if (controllers.get(requestId)?.controller === previous) { + return registerAbortController(requestId, incarnation); + } + return new AbortController(); } /** diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index c41ff16ae8..943ff40fdf 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -62,6 +62,7 @@ import { abortRequest, registerAbortController, deregisterAbortController, + replaceAbortController, tagAbortController, wasFiredOnlyFenced } from "./abort-registry"; @@ -1660,8 +1661,8 @@ export async function runActionInternal< // run. if (ctx.request.incarnation !== currentIncarnation) { currentIncarnation = ctx.request.incarnation; - if (wasFiredOnlyFenced(requestId, registered)) { - registered = registerAbortController(requestId, currentIncarnation); + if (wasFiredOnlyFenced(registered)) { + registered = replaceAbortController(requestId, registered, currentIncarnation); registered.signal.addEventListener("abort", forwardRegisteredAbort, { once: true }); } else { tagAbortController(requestId, registered, currentIncarnation); diff --git a/packages/engine/src/transports/host/createInboundTransportHost.ts b/packages/engine/src/transports/host/createInboundTransportHost.ts index f96c2b2bec..3cae585217 100644 --- a/packages/engine/src/transports/host/createInboundTransportHost.ts +++ b/packages/engine/src/transports/host/createInboundTransportHost.ts @@ -46,7 +46,9 @@ import { import { deregisterAbortController, registerAbortController, - tagAbortController + replaceAbortController, + tagAbortController, + wasFiredOnlyFenced } from "../../execution/abort-registry"; import { generateId } from "../../utils/generate-id"; import { @@ -176,14 +178,30 @@ function isRefusedAdmission(error: unknown): boolean { ); } +/** + * End a request whose run never started. With `expectedIncarnation`, only + * that request is ended: a record another request put under the id since is + * left alone. Without it, whatever holds the id is ended. + * + * The read and the write are separate calls, so a record replaced between + * them is still overwritten. The conditional store write cannot close that: + * it does not write `status`. + */ async function terminateUnenqueuedRequest( stores: StoreRegistry, requestId: string, - status: "failed" | "aborted" = "failed" + status: "failed" | "aborted" = "failed", + expectedIncarnation?: string ): Promise { try { const record = await stores.request.get(requestId); if (record === undefined || isTerminalRequestStatus(record.status)) return; + if ( + expectedIncarnation !== undefined && + resolveRequestIncarnation(record) !== expectedIncarnation + ) { + return; + } const now = Date.now(); await stores.request.set( requestId, @@ -910,7 +928,11 @@ export function createInboundTransportHost( // last-write-wins hand-off the enqueue-time record already uses, so this // adds a window rather than a second registry to keep in sync. The // `finally` below removes it on every exit, started or not. - const queuedAbort = registerAbortController(requestId); + let queuedAbort = registerAbortController(requestId); + // The incarnation of the request this dispatch claimed, once it has. + // Every terminal write below is for that request, not for whatever + // holds the id by the time the write happens. + let claimedIncarnation: string | undefined; // A `queue` run's start is deferred behind the key, so `dispatchLocal` // (which registers `activeRequests` and writes the request record) has // not run when this handle is returned. Materialize a discoverable @@ -940,6 +962,7 @@ export function createInboundTransportHost( // Until its record is claimed this controller belongs to no known // request, so an abort fenced on one does not fire it. Tagged the // moment the record exists, because a cancel can land from then on. + claimedIncarnation = incarnation; tagAbortController(requestId, queuedAbort, incarnation); }) ) @@ -960,7 +983,7 @@ export function createInboundTransportHost( // admission never touched the foreign owner's, nor the caller's own // earlier request under the id it reused. if (!isRefusedAdmission(error)) { - await terminateUnenqueuedRequest(stores, requestId); + await terminateUnenqueuedRequest(stores, requestId, "failed", claimedIncarnation); } throw error; }); @@ -1022,11 +1045,36 @@ export function createInboundTransportHost( // corrupting outcome rather than an untidy one, so the wait is // registered (above) and the decision is re-read here, at the last // moment before anything runs (FIX-1077). + // + // A fenced fire was for the request this dispatch claimed. If + // another request has taken the id since, that fire is not its + // cancel: the run adopts it and starts on an unfired controller, + // and its own start read settles any cancel recorded on it. An + // unfenced fire (shutdown) stops whatever holds the id. if (queuedAbort.signal.aborted) { - await terminateUnenqueuedRequest(stores, requestId, "aborted"); - throw new Error( - `Request "${requestId}" was cancelled before it left the concurrency queue` - ); + const holder = await stores.request.get(requestId).catch(() => undefined); + const heldByAnother = + holder !== undefined && + claimedIncarnation !== undefined && + resolveRequestIncarnation(holder) !== claimedIncarnation; + const fencedOnly = wasFiredOnlyFenced(queuedAbort); + if (holder !== undefined && heldByAnother && fencedOnly) { + queuedAbort = replaceAbortController( + requestId, + queuedAbort, + resolveRequestIncarnation(holder) + ); + } else { + await terminateUnenqueuedRequest( + stores, + requestId, + "aborted", + fencedOnly ? claimedIncarnation : undefined + ); + throw new Error( + `Request "${requestId}" was cancelled before it left the concurrency queue` + ); + } } // The check above is not sufficient on its own and is not meant to // be: an abort landing after it would be lost, because `runAction` @@ -1046,8 +1094,13 @@ export function createInboundTransportHost( // a record that was never ours. A session another user created // under the id while the run waited is refused the same way, and // the stub it leaves is ours to settle. - if ( - error instanceof ConcurrencyQueueTimeoutError || + // + // A timeout means no run started, so it ends only the request + // this dispatch claimed. A binding refusal comes from the run, + // which adopted whatever held the id, so it ends that. + if (error instanceof ConcurrencyQueueTimeoutError) { + await terminateUnenqueuedRequest(stores, requestId, "failed", claimedIncarnation); + } else if ( error instanceof FlowInstanceBindingMismatchError || error instanceof UserBindingMismatchError ) { @@ -1110,6 +1163,7 @@ export function createInboundTransportHost( // already went out. The concurrency gate does not apply here — external // dispatch is unarbitrated in v1 (FIX-830). const ts = Date.now(); + let claimedIncarnation: string | undefined; const acceptance = admitOwnership(flow, dispatchEnvelope) .then((admitted) => materializeOwned(flow, dispatchEnvelope, admitted, { @@ -1125,6 +1179,8 @@ export function createInboundTransportHost( startedAt: ts, lastHeartbeatAt: ts, queuedAt: ts + }, (incarnation) => { + claimedIncarnation = incarnation; }) ) .then(() => { @@ -1141,7 +1197,7 @@ export function createInboundTransportHost( // the record stuck in_progress forever. A refused admission wrote // nothing and terminates nothing — the record it found is not ours. if (!isRefusedAdmission(error)) { - await terminateUnenqueuedRequest(stores, requestId); + await terminateUnenqueuedRequest(stores, requestId, "failed", claimedIncarnation); } throw error; }); diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index 542314cb08..be4c734e64 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -2784,6 +2784,113 @@ describe("a queued request cancelled through the route", () => { expect(executed).toEqual(["started cancelled"]); expect((await stores.request.get(requestId))?.status).toBe("aborted"); }); + + // The other order: the queued request is cancelled first, which fires the + // host's controller, and only then does its record go and a replacement by + // the same owner take the id. The fire was for the earlier request. The + // replacement is not cancelled, so the gate must neither end it nor start + // it cancelled; the run adopts it and runs it. + it("does not end the request that took the id since the queued one was cancelled", async () => { + const stores = createInMemoryStores(); + const executed: string[] = []; + let releaseHold!: () => void; + const hold = new Promise((resolve) => { + releaseHold = resolve; + }); + let markHolding!: () => void; + const holding = new Promise((resolve) => { + markHolding = resolve; + }); + const concurrency = { policy: "queue", key: "session" } as const; + const registry = createFlowRegistry(); + registry.register( + defineFlow({ + kind: "queued-abort-replaced", + request: { heartbeatIntervalMs: 0 }, + actions: { + hold: { + inputSchema: z.object({}), + concurrency, + block: handler({ + name: "hold", + inputSchema: z.object({}), + execute: async () => { + markHolding(); + await hold; + return {}; + } + }) + }, + queued: { + inputSchema: z.object({}), + concurrency, + block: handler({ + name: "queued", + inputSchema: z.object({}), + execute: (_input, ctx) => { + executed.push(ctx.signal.aborted ? "started cancelled" : "started live"); + return {}; + } + }) + } + } + })({ id: "queued-abort-replaced" }) + ); + const host = createInboundTransportHost({ + registry, + stores, + resolvePrincipal: defaultBodyUserIdPrincipalResolver, + runtimeConfig: {} + }); + const dispatch = (action: "hold" | "queued", requestId: string) => + host.dispatch({ + source: "http", + flowKind: "queued-abort-replaced", + action, + input: {}, + sessionId: "s_queued_replaced", + requestId, + orgId: DEFAULT_ORG_ID, + principal: { userId: "u_queued", orgId: DEFAULT_ORG_ID } + }); + const requestId = "req_queued_replaced"; + + const holder = dispatch("hold", "req_queued_replaced_holder"); + await holding; + const queued = dispatch("queued", requestId); + for (let i = 0; i < 100 && (await stores.request.get(requestId)) === undefined; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await new Promise((resolve) => setTimeout(resolve, 20)); + + // Cancel the queued request: fired here, on the host's tagged controller. + const response = await handleAbortRequest( + new Request(`http://localhost/api/flows/queued-abort-replaced/requests/${requestId}/abort`, { + method: "POST" + }), + { kind: "abort_request", flowKind: "queued-abort-replaced", requestId }, + { stores } + ); + expect(response.status).toBe(204); + + // Then its record goes and an uncancelled replacement takes the id. + const cancelledRecord = (await stores.request.get(requestId))!; + await stores.request.delete(requestId); + await stores.request.set( + requestId, + { ...cancelledRecord, abortRequested: undefined, incarnation: "inc_replacement" }, + "absent" + ); + + releaseHold(); + await holder.finished; + await queued.finished.catch(() => undefined); + + const after = await stores.request.get(requestId); + expect(after?.incarnation).toBe("inc_replacement"); + expect(after?.status).not.toBe("aborted"); + expect(executed).toEqual(["started live"]); + }); }); // The run's controller must carry the incarnation of the request the run @@ -3366,7 +3473,10 @@ describe("a run's abort controller carries the incarnation it executes as", () = // Every order of fires that can land on the run's controller before the // context settles which request the run is, crossed with whether the - // context adopts the request admission read or another that took the id. + // context adopts the request admission read or another that took the id, + // and with whether another run under the id displaced this run's controller + // from the registry before it settled. How the controller was fired is a + // fact about the controller, so displacement must not change the outcome. // The run starts cancelled exactly when an unfenced fire landed (it was for // whatever runs under the id) or a fenced fire targeted the request the run // turned out to be. Before settling, a fenced fire can only target the @@ -3381,16 +3491,24 @@ describe("a run's abort controller carries the incarnation it executes as", () = ["a fenced then an unfenced fire", ["fenced", "unfenced"]], ["an unfenced then a fenced fire", ["unfenced", "fenced"]] ]; - const cases = sequences.flatMap(([label, fires]) => [ - { label, fires, adopts: "same" as const, startsAborted: fires.length > 0 }, - { label, fires, adopts: "another" as const, startsAborted: fires.includes("unfenced") } - ]); + const cases = sequences.flatMap(([label, fires]) => + (["registered", "displaced"] as const).flatMap((slot) => [ + { label, fires, slot, adopts: "same" as const, startsAborted: fires.length > 0 }, + { + label, + fires, + slot, + adopts: "another" as const, + startsAborted: fires.includes("unfenced") + } + ]) + ); it.each(cases)( - "$label, adopting the $adopts request: starts aborted = $startsAborted", - async ({ fires, adopts, startsAborted }) => { + "$label, controller $slot, adopting the $adopts request: starts aborted = $startsAborted", + async ({ fires, slot, adopts, startsAborted }) => { const stores = createInMemoryStores(); - const requestId = `req_presettle_${fires.join("_") || "none"}_${adopts}`; + const requestId = `req_presettle_${fires.join("_") || "none"}_${slot}_${adopts}`; const seen: { aborted?: boolean } = {}; const kind = "presettle-fires"; const flow = defineFlow({ @@ -3440,22 +3558,30 @@ describe("a run's abort controller carries the incarnation it executes as", () = await stores.request.delete(requestId); await stores.request.set(requestId, other, "absent"); } + if (slot === "displaced") { + // Another run under the id registers its own controller. + registerAbortController(requestId, "inc_another_run"); + } } } return get(id); }; - await runAction({ - flow, - actionName: "run", - input: {}, - requestId, - userId: "u_presettle", - orgId: DEFAULT_ORG_ID, - stores, - runtimeConfig: {} - }); - stores.request.get = get; + try { + await runAction({ + flow, + actionName: "run", + input: {}, + requestId, + userId: "u_presettle", + orgId: DEFAULT_ORG_ID, + stores, + runtimeConfig: {} + }); + } finally { + stores.request.get = get; + deregisterAbortController(requestId); + } // Precondition: every fire reached this run's controller. expect(landed).toEqual(fires.map(() => true)); From 86d04f20a0d2955f6057a48c5bfcd2e89bb8ac33 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 04:24:26 +0000 Subject: [PATCH 12/12] fix(engine): hand the queued controller to the run, not its signal (FIX-1654) The host's queued path now hands runAction the controller it registered while the request waited, with the incarnation it was tagged with, instead of forwarding its raw signal. The run registers that controller as its own and applies the same rule as at settle: a controller fired only by fenced fires for another incarnation is dropped for a fresh one. A cancel of the request the gate handed over no longer stops a later request that took the id before the run adopted it. Documents the registry/run-local/side-chain controller topology and the settle-gated forwarding in docs/architecture/execution-and-errors.md. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G --- docs/architecture/execution-and-errors.md | 40 ++++- .../engine/src/execution/abort-registry.ts | 11 +- packages/engine/src/execution/runAction.ts | 14 +- packages/engine/src/execution/types.ts | 10 ++ .../host/createInboundTransportHost.ts | 49 +++--- .../transports/host/in-process-dispatcher.ts | 3 + packages/engine/test/abort.test.ts | 156 +++++++++++++++++- 7 files changed, 243 insertions(+), 40 deletions(-) diff --git a/docs/architecture/execution-and-errors.md b/docs/architecture/execution-and-errors.md index 5b9f009022..a50d8b0a77 100644 --- a/docs/architecture/execution-and-errors.md +++ b/docs/architecture/execution-and-errors.md @@ -183,27 +183,44 @@ pipeline ### Work queue signal lifecycle -Background `.sideChain()` tasks are decoupled from the request's transport-level abort signal (FIX-663). Each request constructs two `AbortController`s: +Background `.sideChain()` tasks are decoupled from the request's transport-level abort signal (FIX-663). Each request has three `AbortController`s: -- `abortController` — the abort-registry controller. Fires on an explicit cancellation only, never on a transport signal. Two paths reach it and they converge here: the `/abort` endpoint / `session.abortRequest()` when the request is running in this process, and `runAction`'s heartbeat-tick poll when the intent was recorded by another process (FIX-1026). A cross-process abort is therefore indistinguishable downstream from a local one. -- `sideChainController` — fires only when `abortController` fires. +- `registered`: the abort-registry controller, tagged with the incarnation of the request the run executes as. It fires on an explicit cancellation only, never on a transport signal. Three paths reach it: + - the `/abort` endpoint, when the request is running in this process. The fire is fenced on the incarnation it cancelled, so it does not reach a later request under the same id; + - the run's own reads of the request store: once as it starts, then on each heartbeat tick, when the intent was recorded by another process (FIX-1026). Both reads are fenced the same way; + - an unfenced fire (host shutdown, the CLI stopping its turn), which stops whatever runs under the id. + + A cross-process abort is therefore indistinguishable downstream from a local one. On the host's queued path, `registered` is the controller the host registered while the request waited, handed to the run rather than replaced, so a cancel that landed on it in between is kept. +- `abortController`: the run's own controller. It fires when `registered` fires, but only once the run has settled which request it executes as (see below). +- `sideChainController`: fires only when `abortController` fires. ``` runActionInternal - abortController ← registry; fires on /abort here, or on the - heartbeat poll reading abortRequested from - the request store (cross-process delivery) + registered ← registry, tagged with the run's incarnation; + fires on /abort here (fenced), on the start or + heartbeat read of abortRequested (fenced), or on + shutdown (unfenced). May be the host's handed-over + controller. + abortController ← run-local; registered's fire is forwarded to it + only after the incarnation settles composedSignal = AbortSignal.any([options.signal, abortController.signal]) ← foreground chain; also fires on transport signal - sideChainController ← NEW; listens on abortController.signal ({ once: true }) + sideChainController ← listens on abortController.signal ({ once: true }) does NOT see options.signal / composedSignal createExecutionContext({ signal: composedSignal, sideChainSignal: sideChainController.signal }) + adopts whatever request holds the id now root ctx.signal = composedSignal root ctx._requestSideChainSignal = sideChainController.signal (re-attached on every child scope in _withExecutionScope) + settle: context's incarnation ≠ the one registered was tagged with? + fired only by fenced fires → drop registered, register a fresh one + otherwise → re-tag registered (an unfenced fire stays) + then forward registered's fire, if any, to abortController + then read abortRequested once for the settled incarnation + sequencer .sideChain(block): taskCtx = { ...ctx, signal: ctx._requestSideChainSignal } executeBlock(block, input, taskCtx, path, { signalOverride: taskCtx.signal }) @@ -213,7 +230,14 @@ runActionInternal Wiring details: -- `sideChainController` listens on `abortController.signal` with `{ once: true }`, plus a defensive `if (signal.aborted)` guard for the registration/abort race. A transport signal composed into `composedSignal` via `AbortSignal.any` does **not** propagate to `sideChainController` because the listener is on `abortController.signal` directly. +- Which request a run executes as is only known once the execution context has read the record. Admission reads it earlier, and another request can take the id in between. Until the context settles it, a fire on `registered` is held back. At the settle point: + - If the context adopted a different incarnation and `registered` was fired only by fenced fires, those fires were for the earlier request. The run drops that controller and registers a fresh one. + - Otherwise `registered` is re-tagged and any fire stays, since an unfenced fire applies to whatever runs under the id. + - Then a held fire is forwarded to `abortController`, and the run reads the stored cancel once for the incarnation it settled on. + + How a controller was fired is recorded on the controller itself, so the answer holds even if another run has since taken its registry slot. +- A controller handed over by the host goes through the same rule when the run registers it. If it was fired only by fenced fires aimed at another incarnation than the one admission read, the run starts on a fresh controller instead. +- `sideChainController` listens on `abortController.signal` with `{ once: true }`, plus a defensive `if (signal.aborted)` guard for the registration/abort race. A transport signal composed into `composedSignal` via `AbortSignal.any` does **not** propagate to `sideChainController` because the listener is on `abortController.signal` directly. Neither does a fire on `registered` that the settle step drops. - `_requestSideChainSignal` is an internal `BlockContext` field, propagated through every scope alongside `_requestSideChainPool`. - The sequencer DSL substitutes `ctx.signal` with `_requestSideChainSignal` at `.sideChain()` / `.sideChainIf()` / `.forEachSideChain()` dispatch, and threads a `signalOverride` through `_withExecutionScope` so descendant scopes inherit it rather than the closure-captured root signal. - `drainRequestSideChainPool` takes no signal: it waits unconditionally, on every terminal path — success, `failed`, `aborted`, and `interrupted` alike (FIX-1001). If an explicit `/abort` arrives mid-drain, in-flight tasks self-cancel via their own `ctx.signal` and settle as rejections, so the drain still resolves. **The suspend path is not a terminal path and still does not drain** — see the replay contract below; `suspended` is a pause, and its in-flight background work is re-run after resume. diff --git a/packages/engine/src/execution/abort-registry.ts b/packages/engine/src/execution/abort-registry.ts index bd184fc7ad..a8e3c60705 100644 --- a/packages/engine/src/execution/abort-registry.ts +++ b/packages/engine/src/execution/abort-registry.ts @@ -49,9 +49,16 @@ function matches(entry: Registered, expected: string | undefined): boolean { * `abortRequest` can tell this request from a later one under the same id. * An untagged controller is never fired by a fenced abort; tag it later with * `tagAbortController` once the incarnation is known. + * + * Pass `controller` to register one a caller already holds, such as one handed + * over from an earlier stage of the same dispatch. How it was fired so far + * stays with it. */ -export function registerAbortController(requestId: string, incarnation?: string): AbortController { - const controller = new AbortController(); +export function registerAbortController( + requestId: string, + incarnation?: string, + controller: AbortController = new AbortController() +): AbortController { controllers.set(requestId, incarnation === undefined ? { controller } : { controller, incarnation }); return controller; } diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index 943ff40fdf..202e20f060 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -1449,7 +1449,19 @@ export async function runActionInternal< // run executes as: one that landed earlier, for a request that then lost the // id to the one the context adopted, belongs to that other request and is // dropped with its controller (see the settle step below). - let registered = registerAbortController(requestId, runIncarnation); + // + // A controller handed over by the caller is kept, fires and all, unless its + // only fires were fenced on another request than the one admission read: + // those are not this run's, so it starts on a fresh one. + const handoff = options.abortHandoff; + const keepHandoff = + handoff !== undefined && + !(handoff.incarnation !== runIncarnation && wasFiredOnlyFenced(handoff.controller)); + let registered = registerAbortController( + requestId, + runIncarnation, + keepHandoff ? handoff.controller : undefined + ); const abortController = new AbortController(); const forwardRegisteredAbort = (): void => { if (incarnationSettled) abortController.abort(registered.signal.reason); diff --git a/packages/engine/src/execution/types.ts b/packages/engine/src/execution/types.ts index 894a6d877e..fb1e8391c4 100644 --- a/packages/engine/src/execution/types.ts +++ b/packages/engine/src/execution/types.ts @@ -107,6 +107,16 @@ export type RunActionOptions< */ resolvedActionCore?: ActionCore; signal?: AbortSignal; + /** + * An abort controller the caller registered under `requestId` before the + * run started, with the incarnation it was tagged with. The run registers + * it as its own instead of a fresh one, so a cancel that landed on it is + * not lost in the handoff. A cancel that was aimed at another incarnation + * than the one the run executes as is dropped, the same way the run treats + * its own controller. `signal` is not a substitute: a signal carries a fire + * but not whom it was for. + */ + abortHandoff?: { controller: AbortController; incarnation?: string }; stores: StoreRegistry; retry?: RetryPolicy; responseEmitter?: ResponseEmitter; diff --git a/packages/engine/src/transports/host/createInboundTransportHost.ts b/packages/engine/src/transports/host/createInboundTransportHost.ts index 3cae585217..ec79738912 100644 --- a/packages/engine/src/transports/host/createInboundTransportHost.ts +++ b/packages/engine/src/transports/host/createInboundTransportHost.ts @@ -70,7 +70,6 @@ import { DEFAULT_ORG_ID, isValidOrgId } from "@flow-state-dev/core"; import type { FlowDispatcher, DispatchEnvelope } from "../dispatcher"; import { CLI_SOURCE, INTERNAL_SOURCE, TASK_SOURCE } from "../../execution/transport-sources"; import { - combineSignals, createInProcessDispatcher, isInProcessDispatcher, type InProcessDispatcher @@ -868,26 +867,25 @@ export function createInboundTransportHost( void inProcessAccepted.catch(() => {}); /** - * `cancellation` is a signal the run must inherit rather than merely be - * checked against. The queued branch holds one: an abort that lands - * between its pre-start check and `runAction`'s own - * `registerAbortController` would otherwise be thrown away, because - * `runAction` mints a fresh controller and overwrites the one that was - * aborted. Threading it makes the handoff atomic — there is one signal - * from enqueue to completion, so the abort cannot fall between two - * registrations no matter when it lands (FIX-1077). + * `abortHandoff` is a controller the run must take over rather than + * merely be checked against. The queued branch holds one: an abort that + * lands between its pre-start check and `runAction`'s own registration + * would otherwise be thrown away, because `runAction` would register a + * fresh controller over the one that was aborted. Handing the controller + * itself over makes the handoff atomic, so the abort cannot fall between + * two registrations no matter when it lands (FIX-1077). It is the + * controller and its incarnation, not its signal, so the run can still + * tell a cancel for another request under the id from its own. */ - const startRun = (cancellation?: AbortSignal): Promise => { - const signal = - cancellation === undefined - ? envelope.signal - : envelope.signal === undefined - ? cancellation - : combineSignals(envelope.signal, cancellation); + const startRun = (abortHandoff?: { + controller: AbortController; + incarnation?: string; + }): Promise => { const handle = (effectiveDispatcher as InProcessDispatcher).dispatchLocal( dispatchEnvelope, { - signal, + signal: envelope.signal, + abortHandoff, responseEmitter, effectiveRuntimeConfig: { ...dispatchRuntimeConfig, @@ -1051,6 +1049,7 @@ export function createInboundTransportHost( // cancel: the run adopts it and starts on an unfired controller, // and its own start read settles any cancel recorded on it. An // unfenced fire (shutdown) stops whatever holds the id. + let handoffIncarnation = claimedIncarnation; if (queuedAbort.signal.aborted) { const holder = await stores.request.get(requestId).catch(() => undefined); const heldByAnother = @@ -1059,10 +1058,11 @@ export function createInboundTransportHost( resolveRequestIncarnation(holder) !== claimedIncarnation; const fencedOnly = wasFiredOnlyFenced(queuedAbort); if (holder !== undefined && heldByAnother && fencedOnly) { + handoffIncarnation = resolveRequestIncarnation(holder); queuedAbort = replaceAbortController( requestId, queuedAbort, - resolveRequestIncarnation(holder) + handoffIncarnation ); } else { await terminateUnenqueuedRequest( @@ -1077,12 +1077,13 @@ export function createInboundTransportHost( } } // The check above is not sufficient on its own and is not meant to - // be: an abort landing after it would be lost, because `runAction` - // registers a fresh controller over this one. Handing the signal - // down is what closes that gap — the check short-circuits the run - // entirely when the decision is already made, and the signal - // carries it when it is made a moment later. - return startRun(queuedAbort.signal); + // be: an abort landing after it would be lost if `runAction` + // registered a fresh controller over this one. Handing the + // controller down is what closes that gap: the check + // short-circuits the run entirely when the decision is already + // made, and the controller carries it when it is made a moment + // later, along with whom it was made for. + return startRun({ controller: queuedAbort, incarnation: handoffIncarnation }); }).catch(async (error: unknown) => { // The stub is this dispatch's own by now — materialization // succeeded before the gate opened — so a refusal raised by the diff --git a/packages/engine/src/transports/host/in-process-dispatcher.ts b/packages/engine/src/transports/host/in-process-dispatcher.ts index f9ea66b1f0..db9fb6b6a0 100644 --- a/packages/engine/src/transports/host/in-process-dispatcher.ts +++ b/packages/engine/src/transports/host/in-process-dispatcher.ts @@ -31,6 +31,8 @@ export interface InProcessDispatcherDeps { */ export interface InProcessDispatchContext { signal?: AbortSignal; + /** Forwarded to `runAction`; see `RunActionOptions.abortHandoff`. */ + abortHandoff?: { controller: AbortController; incarnation?: string }; responseEmitter?: ResponseEmitter; /** Effective runtime config with per-flow overrides already merged. */ effectiveRuntimeConfig: RuntimeConfig; @@ -127,6 +129,7 @@ export function createInProcessDispatcher( metadata: envelope.metadata, resolvedActionCore: envelope.resolvedActionCore, signal, + abortHandoff: local.abortHandoff, stores, responseEmitter: local.responseEmitter, runtimeConfig: local.effectiveRuntimeConfig, diff --git a/packages/engine/test/abort.test.ts b/packages/engine/test/abort.test.ts index be4c734e64..790da7aa6e 100644 --- a/packages/engine/test/abort.test.ts +++ b/packages/engine/test/abort.test.ts @@ -2891,6 +2891,139 @@ describe("a queued request cancelled through the route", () => { expect(after?.status).not.toBe("aborted"); expect(executed).toEqual(["started live"]); }); + + // After the gate hands the run a controller for the replacement, that + // request can be cancelled and a third one take the id before the run has + // settled which request it is. The cancel was for the replacement; the run + // adopts the third request, which nobody cancelled, and must run it live. + it("does not stop a later request with a cancel aimed at the one the gate handed over", async () => { + const stores = createInMemoryStores(); + const executed: string[] = []; + let releaseHold!: () => void; + const hold = new Promise((resolve) => { + releaseHold = resolve; + }); + let markHolding!: () => void; + const holding = new Promise((resolve) => { + markHolding = resolve; + }); + const concurrency = { policy: "queue", key: "session" } as const; + const registry = createFlowRegistry(); + registry.register( + defineFlow({ + kind: "queued-abort-handoff", + request: { heartbeatIntervalMs: 0 }, + actions: { + hold: { + inputSchema: z.object({}), + concurrency, + block: handler({ + name: "hold", + inputSchema: z.object({}), + execute: async () => { + markHolding(); + await hold; + return {}; + } + }) + }, + queued: { + inputSchema: z.object({}), + concurrency, + block: handler({ + name: "queued", + inputSchema: z.object({}), + execute: (_input, ctx) => { + executed.push(ctx.signal.aborted ? "started cancelled" : "started live"); + return {}; + } + }) + } + } + })({ id: "queued-abort-handoff" }) + ); + const host = createInboundTransportHost({ + registry, + stores, + resolvePrincipal: defaultBodyUserIdPrincipalResolver, + runtimeConfig: {} + }); + const dispatch = (action: "hold" | "queued", requestId: string) => + host.dispatch({ + source: "http", + flowKind: "queued-abort-handoff", + action, + input: {}, + sessionId: "s_queued_handoff", + requestId, + orgId: DEFAULT_ORG_ID, + principal: { userId: "u_queued", orgId: DEFAULT_ORG_ID } + }); + const requestId = "req_queued_handoff"; + + const holder = dispatch("hold", "req_queued_handoff_holder"); + await holding; + const queued = dispatch("queued", requestId); + for (let i = 0; i < 100 && (await stores.request.get(requestId)) === undefined; i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await new Promise((resolve) => setTimeout(resolve, 20)); + + // The queued request is cancelled, then a replacement takes the id. + const response = await handleAbortRequest( + new Request(`http://localhost/api/flows/queued-abort-handoff/requests/${requestId}/abort`, { + method: "POST" + }), + { kind: "abort_request", flowKind: "queued-abort-handoff", requestId }, + { stores } + ); + expect(response.status).toBe(204); + const cancelledRecord = (await stores.request.get(requestId))!; + await stores.request.delete(requestId); + await stores.request.set( + requestId, + { ...cancelledRecord, abortRequested: undefined, incarnation: "inc_replacement" }, + "absent" + ); + + // Once the gate has handed the run a controller for the replacement, the + // run's next read of the request is where the replacement is cancelled and + // a third request takes the id. + const get = stores.request.get.bind(stores.request); + let firedForReplacement: boolean | undefined; + stores.request.get = async (id: string) => { + if ( + id === requestId && + firedForReplacement === undefined && + hasActiveAbortController(requestId, "inc_replacement") + ) { + firedForReplacement = abortRequest(requestId, "inc_replacement"); + await stores.request.delete(requestId); + await stores.request.set( + requestId, + { ...cancelledRecord, abortRequested: undefined, incarnation: "inc_third" }, + "absent" + ); + } + return get(id); + }; + + try { + releaseHold(); + await holder.finished; + await queued.finished.catch(() => undefined); + } finally { + stores.request.get = get; + } + + // Precondition: the replacement's cancel landed on the controller the gate + // handed over. + expect(firedForReplacement).toBe(true); + const after = await stores.request.get(requestId); + expect(after?.incarnation).toBe("inc_third"); + expect(after?.status).not.toBe("aborted"); + expect(executed).toEqual(["started live"]); + }); }); // The run's controller must carry the incarnation of the request the run @@ -3477,6 +3610,9 @@ describe("a run's abort controller carries the incarnation it executes as", () = // and with whether another run under the id displaced this run's controller // from the registry before it settled. How the controller was fired is a // fact about the controller, so displacement must not change the outcome. + // "handed off" is a controller the caller registered and handed to the run + // (the host's queued path), fired while the run is still reading admission: + // the run must apply the same rule to it as to its own. // The run starts cancelled exactly when an unfenced fire landed (it was for // whatever runs under the id) or a fenced fire targeted the request the run // turned out to be. Before settling, a fenced fire can only target the @@ -3492,7 +3628,7 @@ describe("a run's abort controller carries the incarnation it executes as", () = ["an unfenced then a fenced fire", ["unfenced", "fenced"]] ]; const cases = sequences.flatMap(([label, fires]) => - (["registered", "displaced"] as const).flatMap((slot) => [ + (["registered", "displaced", "handed off"] as const).flatMap((slot) => [ { label, fires, slot, adopts: "same" as const, startsAborted: fires.length > 0 }, { label, @@ -3508,7 +3644,7 @@ describe("a run's abort controller carries the incarnation it executes as", () = "$label, controller $slot, adopting the $adopts request: starts aborted = $startsAborted", async ({ fires, slot, adopts, startsAborted }) => { const stores = createInMemoryStores(); - const requestId = `req_presettle_${fires.join("_") || "none"}_${slot}_${adopts}`; + const requestId = `req_presettle_${fires.join("_") || "none"}_${slot.replace(" ", "_")}_${adopts}`; const seen: { aborted?: boolean } = {}; const kind = "presettle-fires"; const flow = defineFlow({ @@ -3541,14 +3677,23 @@ describe("a run's abort controller carries the incarnation it executes as", () = const other = fresh(); // Admission's read is the first; the context's is the second. The - // fires land just before the context reads. + // fires land just before the context reads, or, for a handed-off + // controller, just before admission reads. + const handoff = + slot === "handed off" + ? { + controller: registerAbortController(requestId, admitted.incarnation), + incarnation: admitted.incarnation + } + : undefined; + const fireOnRead = handoff === undefined ? 2 : 1; const get = stores.request.get.bind(stores.request); let reads = 0; const landed: boolean[] = []; stores.request.get = async (id: string) => { if (id === requestId) { reads += 1; - if (reads === 2) { + if (reads === fireOnRead) { for (const fire of fires) { landed.push( fire === "fenced" ? abortRequest(requestId, admitted.incarnation) : abortRequest(requestId) @@ -3576,7 +3721,8 @@ describe("a run's abort controller carries the incarnation it executes as", () = userId: "u_presettle", orgId: DEFAULT_ORG_ID, stores, - runtimeConfig: {} + runtimeConfig: {}, + abortHandoff: handoff }); } finally { stores.request.get = get;