Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/abort-every-live-attempt.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@flow-state-dev/engine": patch
---

Aborting a request now stops every run of it still live in the process, and `deregisterAbortController` takes an optional controller so one run ending no longer drops another run's (FIX-1665).
104 changes: 70 additions & 34 deletions packages/engine/src/execution/abort-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@
* somewhere else — so a cross-process abort is indistinguishable downstream
* from a local one.
*
* One request id can have several live run attempts at once (the stale-request
* sweep marks a slow but live run `interrupted`, and `/continue` starts another
* under the same id). So the registry holds one controller per live attempt: an
* abort reaches every one of them, and an attempt that ends removes only its
* own, never another attempt's.
*
* A request id names one request only while its record exists, so the id alone
* cannot tell a checked request from a later one that reused it. A controller
* therefore carries the incarnation of the request it belongs to, and a caller
Expand All @@ -25,7 +31,12 @@ interface Registered {
incarnation?: string;
}

const controllers = new Map<string, Registered>();
/** Controllers of the live attempts of each request id, in registration order. */
const controllers = new Map<string, Registered[]>();

function entryOf(requestId: string, controller: AbortController): Registered | undefined {
return controllers.get(requestId)?.find((entry) => entry.controller === controller);
}

/**
* How each controller was fired, kept on the controller rather than on its
Expand All @@ -42,8 +53,12 @@ function matches(entry: Registered, expected: string | undefined): boolean {
}

/**
* Register an AbortController for a request. Returns the controller
* so the caller can use its signal.
* Register an AbortController for a run attempt of a request. Returns the
* controller so the caller can use its signal.
*
* Other attempts' controllers under the same id stay registered beside it.
* Registering a controller that is already registered under the id only
* updates its incarnation.
*
* Pass the request's incarnation when it is known, so a fenced
* `abortRequest` can tell this request from a later one under the same id.
Expand All @@ -59,45 +74,53 @@ export function registerAbortController(
incarnation?: string,
controller: AbortController = new AbortController()
): AbortController {
controllers.set(requestId, incarnation === undefined ? { controller } : { controller, incarnation });
const existing = entryOf(requestId, controller);
if (existing !== undefined) {
if (incarnation === undefined) delete existing.incarnation;
else existing.incarnation = incarnation;
return controller;
}
const entry: Registered = incarnation === undefined ? { controller } : { controller, incarnation };
const entries = controllers.get(requestId);
if (entries === undefined) controllers.set(requestId, [entry]);
else entries.push(entry);
return controller;
}

/**
* Record the incarnation of an already-registered controller. Has no effect
* when `controller` is no longer the one registered under `requestId`, so a
* late tag cannot label a later request's controller.
* when `controller` is no longer registered under `requestId`, so a late tag
* cannot label a controller that has left the registry.
*/
export function tagAbortController(
requestId: string,
controller: AbortController,
incarnation: string
): void {
const entry = controllers.get(requestId);
if (entry !== undefined && entry.controller === controller) {
entry.incarnation = incarnation;
}
const entry = entryOf(requestId, controller);
if (entry !== undefined) entry.incarnation = incarnation;
}

/**
* Signal abort for a request. Returns true if the request was found
* and aborted, false if the request was not in the registry.
* Signal abort for a request: every live attempt's controller under the id.
* Returns true if at least one was found and aborted, false if none was.
*
* With `expectedIncarnation`, fires only when the registered controller
* belongs to that incarnation; a controller of another request under the same
* id, or one not yet tagged, is left alone and the call returns false.
* With `expectedIncarnation`, fires only the controllers that belong to that
* incarnation; a controller of another request under the same id, or one not
* yet tagged, is left alone.
*/
export function abortRequest(requestId: string, expectedIncarnation?: string): boolean {
const entry = controllers.get(requestId);
if (entry === undefined || !matches(entry, expectedIncarnation)) {
return false;
const targets = (controllers.get(requestId) ?? []).filter((entry) =>
matches(entry, expectedIncarnation)
);
for (const entry of targets) {
const fired = firedBy.get(entry.controller) ?? {};
if (expectedIncarnation === undefined) fired.unfenced = true;
else fired.fenced = true;
firedBy.set(entry.controller, fired);
entry.controller.abort();
}
const fired = firedBy.get(entry.controller) ?? {};
if (expectedIncarnation === undefined) fired.unfenced = true;
else fired.fenced = true;
firedBy.set(entry.controller, fired);
entry.controller.abort();
return true;
return targets.length > 0;
}

/**
Expand All @@ -113,32 +136,45 @@ export function wasFiredOnlyFenced(controller: AbortController): boolean {

/**
* Give up `previous` for a fresh, unfired controller tagged `incarnation`.
* The fresh one takes the registry slot only if `previous` still holds it;
* a controller another run displaced does not take the slot back.
* The fresh one takes `previous`'s place only if `previous` is still
* registered; one that has already left the registry is not brought back.
*/
export function replaceAbortController(
requestId: string,
previous: AbortController,
incarnation: string
): AbortController {
if (controllers.get(requestId)?.controller === previous) {
return registerAbortController(requestId, incarnation);
const entry = entryOf(requestId, previous);
const fresh = new AbortController();
if (entry !== undefined) {
entry.controller = fresh;
entry.incarnation = incarnation;
}
return new AbortController();
return fresh;
}

/**
* Remove the controller from the registry. Called on any terminal state.
* Remove a controller from the registry. Called by a run attempt on any
* terminal state, with its own `controller`, so the controllers of other
* attempts still live under the id stay registered. Without `controller`,
* every controller under the id is removed.
*/
export function deregisterAbortController(requestId: string): void {
controllers.delete(requestId);
export function deregisterAbortController(requestId: string, controller?: AbortController): void {
if (controller === undefined) {
controllers.delete(requestId);
return;
}
const entries = controllers.get(requestId);
if (entries === undefined) return;
const remaining = entries.filter((entry) => entry.controller !== controller);
if (remaining.length === 0) controllers.delete(requestId);
else controllers.set(requestId, remaining);
}

/**
* Check whether a request has an active abort controller. With
* `expectedIncarnation`, only a controller of that incarnation counts.
*/
export function hasActiveAbortController(requestId: string, expectedIncarnation?: string): boolean {
const entry = controllers.get(requestId);
return entry !== undefined && matches(entry, expectedIncarnation);
return (controllers.get(requestId) ?? []).some((entry) => matches(entry, expectedIncarnation));
}
15 changes: 10 additions & 5 deletions packages/engine/src/execution/runAction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1694,6 +1694,11 @@ async function runActionAttempt<
const keepHandoff =
handoff !== undefined &&
!(handoff.incarnation !== runIncarnation && wasFiredOnlyFenced(handoff.controller));
// A handed-over controller this run does not keep leaves the registry, so it
// is not left beside the run's own as a second attempt's.
if (handoff !== undefined && !keepHandoff) {
deregisterAbortController(requestId, handoff.controller);
}
let registered = registerAbortController(
requestId,
runIncarnation,
Expand Down Expand Up @@ -2002,7 +2007,7 @@ async function runActionAttempt<
// rather than leave a client polling forever (FIX-1511). Then clean up
// the abort controller / heartbeat and rethrow so `finished` rejects.
await recoverFromPreTransitionFailure();
deregisterAbortController(requestId);
deregisterAbortController(requestId, registered);
if (heartbeatTimer !== undefined) clearInterval(heartbeatTimer);
if (!isReplayMode) {
// The failed record is this run's terminal write; the run's exit
Expand Down Expand Up @@ -2125,7 +2130,7 @@ async function runActionAttempt<
}, ctx, { internalSeams, logger });
} catch (startupError) {
stopHeartbeatTimer();
deregisterAbortController(requestId);
deregisterAbortController(requestId, registered);
throw startupError;
}

Expand Down Expand Up @@ -2254,7 +2259,7 @@ async function runActionAttempt<
durationMs: Date.now() - startedAt
});

deregisterAbortController(requestId);
deregisterAbortController(requestId, registered);
// Another run of this request still live in this process keeps the
// shared registry entry; it deregisters when it ends.
if (attempt.end()) await registry.deregister(requestId).catch(() => {});
Expand Down Expand Up @@ -2556,7 +2561,7 @@ async function runActionAttempt<

// Deregister abort controller and active registry. An unstamped run stays
// registered, heartbeat stopped, for the stale-request sweep to stamp.
deregisterAbortController(requestId);
deregisterAbortController(requestId, registered);
if (finalized) {
await registry.deregister(requestId).catch((err) => {
logRuntimeEvent(logger, "warn", "[flow-state] registry deregister failed", {
Expand Down Expand Up @@ -2842,7 +2847,7 @@ async function runActionAttempt<

// Deregister abort controller and active registry. An unstamped run stays
// registered, heartbeat stopped, for the stale-request sweep to stamp.
deregisterAbortController(requestId);
deregisterAbortController(requestId, registered);
if (finalized) {
await registry.deregister(requestId).catch((err) => {
logRuntimeEvent(logger, "warn", "[flow-state] registry deregister failed", {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -947,9 +947,9 @@ export function createInboundTransportHost(
// Registered HERE rather than left to `runAction`, because between this
// dispatch and the run's own registration the request is real,
// discoverable, and cancellable by anyone reading the store — and yet
// has no controller for `abortRequest` to find. `runAction` re-registers
// (overwriting this one) when it actually starts, which is the same
// last-write-wins hand-off the enqueue-time record already uses, so this
// has no controller for `abortRequest` to find. `runAction` takes it
// over when it actually starts (keeping it, or swapping it for a fresh
// one), the same hand-off the enqueue-time record already uses, so this
// adds a window rather than a second registry to keep in sync. The
// `finally` below removes it on every exit, started or not.
let queuedAbort = registerAbortController(requestId);
Expand Down Expand Up @@ -1172,7 +1172,7 @@ export function createInboundTransportHost(
// cancelled, or timed out — so the pre-start window cannot leak
// controllers into a long-lived process. Idempotent with
// `runAction`'s own deregistration on the path where it did start.
deregisterAbortController(requestId);
deregisterAbortController(requestId, queuedAbort);
});
} else {
// Nothing is written before the run here (`runAction` writes its own
Expand Down
37 changes: 36 additions & 1 deletion packages/engine/test/abort.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,9 @@ describe("abort-registry", () => {
expect(controller.signal.aborted).toBe(true);
});

it("does not tag a controller that has since been replaced", () => {
it("does not tag a controller that has since left the registry", () => {
const first = registerAbortController("test-req-1");
deregisterAbortController("test-req-1", first);
const second = registerAbortController("test-req-1", "inc_b");
tagAbortController("test-req-1", first, "inc_a");

Expand All @@ -118,6 +119,39 @@ describe("abort-registry", () => {
expect(controller.signal.aborted).toBe(true);
});
});

describe("several live attempts under one id", () => {
it("fires every attempt's controller", () => {
const first = registerAbortController("test-req-1", "inc_a");
const second = registerAbortController("test-req-1", "inc_a");

expect(abortRequest("test-req-1", "inc_a")).toBe(true);
expect(first.signal.aborted).toBe(true);
expect(second.signal.aborted).toBe(true);
});

it("keeps the other attempt's controller when one attempt deregisters", () => {
const first = registerAbortController("test-req-1", "inc_a");
const second = registerAbortController("test-req-1", "inc_a");
deregisterAbortController("test-req-1", first);

expect(hasActiveAbortController("test-req-1")).toBe(true);
expect(abortRequest("test-req-1")).toBe(true);
expect(first.signal.aborted).toBe(false);
expect(second.signal.aborted).toBe(true);

deregisterAbortController("test-req-1", second);
expect(hasActiveAbortController("test-req-1")).toBe(false);
});

it("registers a controller it already holds only once", () => {
const controller = registerAbortController("test-req-1");
registerAbortController("test-req-1", "inc_a", controller);
deregisterAbortController("test-req-1", controller);

expect(hasActiveAbortController("test-req-1")).toBe(false);
});
});
});

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -411,6 +445,7 @@ describe("handleAbortRequest", () => {
const write = stores.request.setFieldsIfStatus.bind(stores.request);
stores.request.setFieldsIfStatus = async (...args) => {
const result = await write(...args);
deregisterAbortController(REUSED, own);
await stores.request.delete(REUSED);
await stores.request.set(
REUSED,
Expand Down
Loading
Loading