Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix(engine): read a stored cancel only for the request this run execu…
…tes as

The catch-up read after a re-tag, the heartbeat intent poll and both
abort classifications read the cancel flag by id alone, so a cancel
recorded on a later request that took the id could stop or reclassify
this run. Each now checks the flag first (the O(1) read) and, only when
it is set, reads the record and requires its incarnation to match the
one the run executes as. The poll's fire is fenced on it too.

Co-Authored-By: Claude <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HNTQkGXCUryuVD2TEBLi6G
  • Loading branch information
claude committed Sep 30, 2026
commit d0aaad4c6552bd9526882a41b1f48c57cc33eeea
43 changes: 33 additions & 10 deletions packages/engine/src/execution/runAction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1004,6 +1004,24 @@ export async function runActionInternal<
)
))
);
// The incarnation this run executes as: admission's, until the execution
// context adopts the record and says otherwise (see the re-tag below).
let currentIncarnation = runIncarnation;
/**
* Whether a cancel is recorded on the request this run executes as. The flag
* is read first, as the O(1) `isAbortRequested`, and only when it is set is
* the record read, to check it is still this run's request and not a later
* one that took the id. The common path stays one narrow read.
*/
const abortRecordedForThisRun = async (): Promise<boolean> => {
if (!(await options.stores.request.isAbortRequested(requestId))) return false;
const record = await options.stores.request.get(requestId);
return (
record !== undefined &&
record.abortRequested === true &&
resolveRequestIncarnation(record) === currentIncarnation
);
};

await registry.register({
requestId,
Expand Down Expand Up @@ -1068,10 +1086,10 @@ export async function runActionInternal<
if (deliveredAbort || abortPollInFlight) return;
abortPollInFlight = true;
try {
if (!(await options.stores.request.isAbortRequested(requestId))) return;
if (!(await abortRecordedForThisRun())) return;
// Returns false when no controller is registered yet. Not a delivery, so
// the latch stays unset and the next tick retries.
if (!abortRequest(requestId)) return;
if (!abortRequest(requestId, currentIncarnation)) return;
deliveredAbort = true;
logRuntimeEvent(logger, "info", "[flow-state] [abort] cross-process abort delivered", {
requestId
Expand Down Expand Up @@ -1637,9 +1655,10 @@ export async function runActionInternal<
// one this run executes as, so a cancel fenced on it fires here. A cancel
// of that request recorded while the tag was still the earlier one missed
// the controller, so read the stored intent once and deliver it.
if (ctx.request.incarnation !== runIncarnation) {
tagAbortController(requestId, abortController, ctx.request.incarnation);
if (await options.stores.request.isAbortRequested(requestId).catch(() => false)) {
if (ctx.request.incarnation !== currentIncarnation) {
currentIncarnation = ctx.request.incarnation;
tagAbortController(requestId, abortController, currentIncarnation);
if (await abortRecordedForThisRun().catch(() => false)) {
abortController.abort();
}
}
Expand Down Expand Up @@ -2283,7 +2302,10 @@ export async function runActionInternal<
? await options.stores.request.get(requestId).catch(() => undefined)
: undefined;
let wasIntentionalAbort =
signalAborted && (deliveredAbort || classificationRecord?.abortRequested === true);
signalAborted &&
(deliveredAbort ||
(classificationRecord?.abortRequested === true &&
resolveRequestIncarnation(classificationRecord) === currentIncarnation));

// The failure path's normalized error is needed for the client-visible
// error item, which is emitted before the drain so a caller hears the
Expand Down Expand Up @@ -2359,18 +2381,19 @@ export async function runActionInternal<
// controller, while a stop accepted on another instance only sets the
// durable flag and reaches us through the heartbeat's abort poll —
// which is running precisely because the heartbeat outlives the drain.
// `isAbortRequested`, not `get()`: this asks one boolean, and the
// interface requires it be O(1) in item count, where `get()`
// `isAbortRequested` first, not `get()`: this asks one boolean, and
// the interface requires it be O(1) in item count, where `get()`
// materializes the whole item history to answer it on the persistent
// adapters. A store failure resolves to `false`, which keeps the
// adapters. Only a set flag pays for the `get()` that checks it is
// this run's request. A store failure resolves to `false`, which keeps the
// branch already chosen — the same direction the classification read
// above fails in. (That read stays on `get()` deliberately: it is the
// call that absorbs a transient failure before `patchRequestRecord`,
// which does not catch.)
if (!wasIntentionalAbort) {
wasIntentionalAbort =
deliveredAbort ||
(await options.stores.request.isAbortRequested(requestId).catch(() => false));
(await abortRecordedForThisRun().catch(() => false));
}
} finally {
// Clear the heartbeat — the drain is done and the terminal write is
Expand Down
144 changes: 144 additions & 0 deletions packages/engine/test/abort.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2732,4 +2732,148 @@ describe("a run's abort controller carries the incarnation it executes as", () =
expect(record?.incarnation).toBe(adopted.incarnation);
expect(record?.status).toBe("aborted");
});

// The catch-up read after a re-tag answers for whatever holds the id at that
// moment. A cancel recorded on a request that took the id since is not this
// run's, and must not stop it.
it("does not stop the run for a cancel recorded on a request that took the id since", async () => {
const stores = createInMemoryStores();
const requestId = "req_adopted_then_reused";
const seen: { aborted?: boolean } = {};
const flow = defineFlow({
kind: "adopt-fence-reused",
request: { heartbeatIntervalMs: 0 },
actions: {
run: {
inputSchema: z.unknown(),
block: handler({
name: "observe",
inputSchema: z.unknown(),
outputSchema: z.string(),
execute: async (_input: unknown, ctx) => {
seen.aborted = ctx.signal.aborted;
return "ran";
}
})
}
}
})({ id: "adopt-fence-reused" });

const adopted = createInitialRequestRecord(
{
requestId,
flowKind: "adopt-fence-reused",
flowId: "adopt-fence-reused",
actionName: "run",
userId: "u_adopt",
orgId: DEFAULT_ORG_ID
},
Date.now()
);
await stores.request.set(requestId, adopted, "absent");

// Admission read an earlier request under the id, so the run re-tags.
const get = stores.request.get.bind(stores.request);
let first = true;
stores.request.get = async (id: string) => {
const record = await get(id);
if (first && id === requestId && record !== undefined) {
first = false;
return { ...record, incarnation: "inc_admitted_earlier" };
}
return record;
};
// Before the catch-up read lands, the adopted request's record goes and a
// later request takes the id, and someone cancels that later request.
const isAbortRequested = stores.request.isAbortRequested.bind(stores.request);
let swapped = false;
stores.request.isAbortRequested = async (id: string) => {
if (!swapped && id === requestId) {
swapped = true;
await stores.request.delete(requestId);
await stores.request.set(requestId, { ...adopted, incarnation: "inc_later_request" }, "absent");
await stores.request.setFieldsIfStatus(
requestId,
{ abortRequested: true },
["in_progress"],
Date.now(),
"inc_later_request"
);
}
return isAbortRequested(id);
};

await runAction({
flow,
actionName: "run",
input: {},
requestId,
userId: "u_adopt",
orgId: DEFAULT_ORG_ID,
stores,
runtimeConfig: {}
});
stores.request.get = get;
stores.request.isAbortRequested = isAbortRequested;

expect(swapped).toBe(true);
expect(seen.aborted).toBe(false);
});

// The heartbeat's intent poll reads by id too, so it has the same question to
// answer: a cancel recorded on a request that took the id is not this run's.
it("does not let the heartbeat poll deliver a later request's cancel", async () => {
const stores = createInMemoryStores();
const requestId = "req_poll_reused";
const seen: { aborted?: boolean } = {};
const flow = defineFlow({
kind: "poll-fence",
request: { heartbeatIntervalMs: 10 },
actions: {
run: {
inputSchema: z.unknown(),
block: handler({
name: "park",
inputSchema: z.unknown(),
outputSchema: z.string(),
execute: async (_input: unknown, ctx) => {
// The run's record goes and a later request takes the id, and
// someone cancels that later request.
const mine = (await stores.request.get(requestId))!;
await stores.request.delete(requestId);
await stores.request.set(
requestId,
{ ...mine, incarnation: "inc_later_request" },
"absent"
);
await stores.request.setFieldsIfStatus(
requestId,
{ abortRequested: true },
["in_progress"],
Date.now(),
"inc_later_request"
);
// Several poll ticks.
await new Promise((resolve) => setTimeout(resolve, 100));
seen.aborted = ctx.signal.aborted;
return "ran";
}
})
}
}
})({ id: "poll-fence" });

await runAction({
flow,
actionName: "run",
input: {},
requestId,
userId: "u_poll",
orgId: DEFAULT_ORG_ID,
stores,
runtimeConfig: {}
});

expect(seen.aborted).toBe(false);
});
});
Loading