diff --git a/.changeset/held-notice-after-crash.md b/.changeset/held-notice-after-crash.md new file mode 100644 index 0000000000..338b42ea43 --- /dev/null +++ b/.changeset/held-notice-after-crash.md @@ -0,0 +1,5 @@ +--- +"@flow-state-dev/engine": patch +--- + +A task's notice held for the request that filed it no longer waits out its resend backoff when that request's server died mid-run. `requestHost.waits.running` now answers `false` for a request still recorded `in_progress` whose registry heartbeat has been silent past the host's stale-request threshold (where the flow's heartbeat is on and beats at least twice per threshold), and the stale-request and startup sweeps bring the wakes kept for the session of each request they mark interrupted forward to now, so the filer's pass sends the notice at once. With the defaults, the notice is heard within about 90 seconds of the request's last heartbeat. `detectInterruptedRequests` and `createStaleRequestSweeper` take an optional `wakeEntriesOf` for this (FIX-2166). diff --git a/apps/docs/docs/orchestration/task-board.md b/apps/docs/docs/orchestration/task-board.md index 4e6435dff3..18f98c56e7 100644 --- a/apps/docs/docs/orchestration/task-board.md +++ b/apps/docs/docs/orchestration/task-board.md @@ -798,6 +798,8 @@ When a task the board handed out ends, the session that filed it is told, once. While the request that filed the task is still running, its notice is held. When that request ends, it sends what it held right away. When it parks, on a person's approval or input or on a [`getTask` wait](#waiting-on-a-task), it holds nothing more. A task that ends during the park sends its notice right away. A notice held while the request ran goes out about two seconds after the park. So a turn that files a task and never waits on it hears the ending once that turn ends, or soon after it parks. Holding needs a host with durable execution; elsewhere a notice is sent as soon as the task ends. +If the server stops while the request is running, the held notice goes out when the request is marked `interrupted`. With default settings that is within about 90 seconds of the request's last heartbeat, or a few seconds after the server comes back if it's down longer. `staleSweepThresholdMs` and `staleSweepIntervalMs` set that window; see [connection resilience](../server/connection-resilience.md). + A task that fails and is retried sends nothing until its last attempt ends. These endings send no notice either: - A task the filer [waited on](#waiting-on-a-task) with `getTask({ id, wait: true })`, either a wait that was open when the task ended, or one started after it ended but before an `onTaskSettled` request for that ending had started. The `getTask` call gets the result instead. A wait that reached its deadline first doesn't count: that ending goes to the request's next `getTask` only while the request is still running. If the request parks or ends first, the ending is sent as a notice. diff --git a/packages/core/src/types/request-host.ts b/packages/core/src/types/request-host.ts index b148972266..77f0381872 100644 --- a/packages/core/src/types/request-host.ts +++ b/packages/core/src/types/request-host.ts @@ -198,7 +198,11 @@ export interface RequestWaits { * Whether each request is running its own work now (`in_progress`): neither * parked (`suspended`) nor ended. `false` for one parked * on a person or a wait, for one that ended or whose process died, for a - * request of another session, and for one with no record. A board reads it + * request of another session, and for one with no record. Also `false` for + * one still recorded `in_progress` whose registry heartbeat has been silent + * past the host's stale-request threshold: its process died before the sweep + * marked it interrupted (FIX-2166). Where heartbeats can't tell (off, or + * slower than half the threshold), the record alone answers. A board reads it * to hold a task's notice while the request that filed the task still runs * (FIX-2123), and not while that request is parked. */ diff --git a/packages/engine/src/context/create-request-host.ts b/packages/engine/src/context/create-request-host.ts index 8c56338e8f..7d5948c1de 100644 --- a/packages/engine/src/context/create-request-host.ts +++ b/packages/engine/src/context/create-request-host.ts @@ -93,7 +93,7 @@ import { type SessionRecordSeed } from "./session-birth"; import { pinRejectsCaller } from "./instance-pin"; -import { evaluateLivenessGate, type LivenessGateInputs } from "./liveness-gate"; +import { evaluateLivenessGate, heartbeatStaleThreshold, type LivenessGateInputs } from "./liveness-gate"; import { readLiveness } from "./liveness-read"; import type { WaitResumeOperation } from "./wait-resume-operation"; import { createRequestWakes } from "../durability/request-wakes"; @@ -788,7 +788,13 @@ export function createRequestHost(inputs: RequestHostInputs): RequestHostBuild { // deadline is a wake, so it is present only beside both. The user and the // request come from here, never the block. if (inputs.waits !== undefined && waitResume !== undefined && inputs.wakes !== undefined) { + // `running` answers for requests of this session only, which run this + // flow, so this flow's heartbeat is theirs (FIX-2166). + const staleThresholdMs = heartbeatStaleThreshold(inputs.liveness); (host as { waits?: RequestHost["waits"] }).waits = createRequestWaits({ + ...(staleThresholdMs !== undefined + ? { heartbeat: { readEntry: (requestId) => stores.activeRequests.get(requestId), staleThresholdMs, now: nowMs } } + : {}), provider: inputs.waits.provider, deadlines: inputs.waits.deadlines, userId: identity.userId, diff --git a/packages/engine/src/context/liveness-gate.ts b/packages/engine/src/context/liveness-gate.ts index 3e71f37e61..02fc146036 100644 --- a/packages/engine/src/context/liveness-gate.ts +++ b/packages/engine/src/context/liveness-gate.ts @@ -65,6 +65,28 @@ export type LivenessGateVerdict = const DEFAULT_HEARTBEAT_MS = 10_000; const DEFAULT_STALE_THRESHOLD_MS = 60_000; +/** + * The stale threshold a request's heartbeat can be read against, or + * `undefined` where heartbeats can't say a request stopped (FIX-2166): they + * are off (`0`), or beat slower than half the threshold, so a healthy request + * can look stale between beats. The same rule as the gate's second arm, and + * the same defaults. Unlike the gate it reads one process's own registry + * entry as well as a shared one's: a stale entry is a stopped heartbeat + * wherever it is kept. And it ignores the gate's third arm, the sweep's + * cadence, deliberately: with sweeping off nothing marks a dead request + * interrupted, and this read is then the only thing that frees a notice held + * for it. + */ +export function heartbeatStaleThreshold( + inputs: Pick +): number | undefined { + const heartbeatMs = inputs.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_MS; + const thresholdMs = inputs.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS; + if (!Number.isFinite(heartbeatMs) || heartbeatMs <= 0) return undefined; + if (!Number.isFinite(thresholdMs) || thresholdMs < 2 * heartbeatMs) return undefined; + return thresholdMs; +} + /** * Decide whether the liveness verb can be trusted in this deployment. * diff --git a/packages/engine/src/context/liveness-read.ts b/packages/engine/src/context/liveness-read.ts index edadc9d817..d32c0896f2 100644 --- a/packages/engine/src/context/liveness-read.ts +++ b/packages/engine/src/context/liveness-read.ts @@ -83,7 +83,11 @@ export type LivenessReadInputs = { * wrong question, so it is answered before the comparison rather than by tuning * it — see `queuedAt` on `ActiveRequestEntry` (FIX-999). */ -function isFresh(entry: ActiveRequestEntry, nowMs: number, staleThresholdMs: number): boolean { +export function isFresh( + entry: Pick, + nowMs: number, + staleThresholdMs: number +): boolean { // Accepted into an external queue, not yet claimed. No worker exists to beat // for it, so its age is queue wait, not death — it is live by construction. // diff --git a/packages/engine/src/durability/request-wakes.ts b/packages/engine/src/durability/request-wakes.ts index 7e593ef361..0af07c1f8f 100644 --- a/packages/engine/src/durability/request-wakes.ts +++ b/packages/engine/src/durability/request-wakes.ts @@ -219,3 +219,38 @@ export function createRequestWakes(inputs: RequestWakesInputs): RequestWakes { } }; } + +/** + * Bring a session's kept wakes forward to `now`, for a request in it the stale + * sweep just marked interrupted (FIX-2166). What a pass held for that request + * while it ran is owed now that it never will end on its own, so the pass is + * owed now rather than at its backoff. A wake's entry pays only what is due, + * so a wake fired early costs one pass and moves itself on. + * + * One read per entry the session's flow declares and one write per wake kept + * there that is due later; nothing else is read. The write is an `owe` from + * the wake just read: it keeps the earlier due time and bumps the version, so + * a pass that read the wake before this write and settles it after finds it + * changed (`stale`) and leaves it due now, rather than moving it back to its + * backoff. A wake a pass cleared between the read and the write is kept + * again, due now: its fire finds nothing owed and clears it. + * + * @param entries The internal entries of the session's flow: the only keys a + * wake in it can have. + */ +export async function bringSessionWakesForward( + store: WakeStore, + session: { readonly tenantId?: string | null; readonly sessionId: string }, + entries: readonly string[], + now: number +): Promise { + const tenant = session.tenantId != null && session.tenantId.length > 0 ? session.tenantId : undefined; + for (const entry of entries) { + const key = { ...(tenant !== undefined ? { tenantId: tenant } : {}), sessionId: session.sessionId, entry }; + const wake = await store.get(key); + if (wake === null || wake.dueAt <= now) continue; + const { version: _version, createdAt: _createdAt, updatedAt: _updatedAt, ...owed } = wake; + const kept = await store.owe({ ...owed, dueAt: now }); + noteWakeOwed(store, kept.dueAt); + } +} diff --git a/packages/engine/src/durability/wait-chain.ts b/packages/engine/src/durability/wait-chain.ts index dd3532be41..bde6fe2571 100644 --- a/packages/engine/src/durability/wait-chain.ts +++ b/packages/engine/src/durability/wait-chain.ts @@ -45,7 +45,8 @@ import { type WaitOutcome, type WaitRow } from "@flow-state-dev/core/types"; -import type { RequestRecord, StoreRegistry } from "../stores/types"; +import type { ActiveRequestEntry, RequestRecord, StoreRegistry } from "../stores/types"; +import { isFresh } from "../context/liveness-read"; import type { DurabilityProvider } from "./types"; import { resumeWaitGate, type WaitResumeDeps } from "./resume-wait-gate"; import { drivenAgain } from "./stop-suspended"; @@ -227,7 +228,48 @@ export function createRequestWaits(inputs: { readonly sessionId: string; /** A request's record, with its item log, from the request store. */ readonly readRequest: (requestId: string) => Promise | undefined>; + /** + * How `running` tells a request whose process died from one still running + * (FIX-2166): its registry entry's heartbeat, read against the host's stale + * threshold, the one past which the stale-request sweep marks it + * interrupted. Absent where heartbeats can't say (they are off, or beat + * slower than half the threshold): there `running` reads the record alone. + * + * The threshold is per process, as the sweep's is: each process reads with + * its own `staleSweepThresholdMs` and the heartbeat interval of the flow + * version it runs, and its sweep trusts the same values. A process + * configured tighter than the one running the request, or a deploy that + * slows the heartbeat, can read a live request as stopped, exactly as its + * sweep would reap it. + * + * The read stays on with sweeping off (`staleSweepIntervalMs: 0`), on + * purpose: nothing then marks a dead request interrupted, so this read is + * the only thing that frees a notice held for it. + */ + readonly heartbeat?: { + readonly readEntry: (requestId: string) => Promise | undefined>; + readonly staleThresholdMs: number; + readonly now: () => number; + }; }): RequestWaits { + // An `in_progress` record whose heartbeat stopped past the stale threshold + // is a request whose process died before the sweep marked it interrupted + // (FIX-2166). No entry reads as the record says: a request with no + // registration is no proof that its process is gone, nor is an entry with + // no heartbeat recorded (a custom registry's), nor a registry that can't be + // read: each fails safe, to the record's answer. + const stillBeating = async (requestId: string): Promise => { + const heartbeat = inputs.heartbeat; + if (heartbeat === undefined) return true; + let entry: Pick | undefined; + try { + entry = await heartbeat.readEntry(requestId); + } catch { + return true; + } + if (entry === undefined || entry.lastHeartbeatAt == null) return true; + return isFresh(entry, heartbeat.now(), heartbeat.staleThresholdMs); + }; return { deadlines: inputs.deadlines, depth: ({ binding, own }) => @@ -256,7 +298,11 @@ export function createRequestWaits(inputs: { for (const requestId of new Set(requestIds)) { try { const record = await inputs.readRequest(requestId); - answers[requestId] = record !== undefined && record.sessionId === inputs.sessionId && RUNNING.has(record.status); + answers[requestId] = + record !== undefined && + record.sessionId === inputs.sessionId && + RUNNING.has(record.status) && + (await stillBeating(requestId)); } catch { // A record that can't be read answers for itself only: not running, so nothing waits on it. answers[requestId] = false; diff --git a/packages/engine/src/execution/request-recovery.ts b/packages/engine/src/execution/request-recovery.ts index 5d91408db8..2814a092df 100644 --- a/packages/engine/src/execution/request-recovery.ts +++ b/packages/engine/src/execution/request-recovery.ts @@ -17,6 +17,7 @@ import { logRuntimeEvent, type RuntimeLogger, DEFAULT_RUNTIME_LOGGER } from "./l import { settledRecordFields } from "./request-action-result"; import { runAction } from "./runAction"; import { resolveRecordOwner } from "../context/record-owner"; +import { bringSessionWakesForward } from "../durability/request-wakes"; /** * The heartbeat-age threshold {@link detectInterruptedRequests} applies when its @@ -25,6 +26,19 @@ import { resolveRecordOwner } from "../context/record-owner"; */ export const DEFAULT_DETECTION_STALE_THRESHOLD_MS = 30_000; +/** + * {@link detectInterruptedRequests}' `wakeEntriesOf`, read off a flow registry + * (FIX-2166): the internal entries of the flow instance that owns a record. + * Read per call, so a flow registered after the sweep was built is found. A + * record whose owner isn't registered here has none. + */ +export function internalEntriesFrom(registry: Pick): (record: RequestRecord) => readonly string[] { + return (record) => { + const owner = resolveRecordOwner(registry, record); + return owner.ok ? Object.keys(owner.flow.internal?.actions ?? {}) : []; + }; +} + export type InterruptedRequestInfo = { entry: ActiveRequestEntry; requestRecord?: RequestRecord; @@ -68,6 +82,14 @@ export async function detectInterruptedRequests(options: { userId?: string; /** Restrict the sweep to entries this predicate admits. Undefined means unrestricted. */ ownedBy?: (entry: ActiveRequestEntry) => boolean; + /** + * The internal entries of the flow a request ran on (FIX-2166): the keys its + * session's wakes can have. With it, a request marked interrupted here + * brings its session's wakes forward to now, so a pass that held something + * for the request while it ran sends it at once, not at its backoff. + * Undefined, or no wake store, leaves the wakes as they are. + */ + wakeEntriesOf?: (record: RequestRecord) => readonly string[]; logger?: RuntimeLogger; }): Promise { const { stores, userId, ownedBy, logger = DEFAULT_RUNTIME_LOGGER } = options; @@ -86,6 +108,20 @@ export async function detectInterruptedRequests(options: { ); const results: InterruptedRequestInfo[] = []; + /** Owe the interrupted request's session its passes now (FIX-2166); never fails the sweep. */ + const wakeSessionOf = async (record: RequestRecord, now: number): Promise => { + const wakes = stores.wakes; + if (wakes === undefined || options.wakeEntriesOf === undefined || record.sessionId == null) return; + try { + await bringSessionWakesForward(wakes, { tenantId: record.tenantId, sessionId: record.sessionId }, options.wakeEntriesOf(record), now); + } catch (error) { + logRuntimeEvent(logger, "warn", "[flow-state] an interrupted request's session wakes weren't brought forward: they fire at their own due time", { + requestId: record.id, + error: error instanceof Error ? error.message : String(error) + }); + } + }; + for (const entry of stale) { const requestRecord = await stores.request.get(entry.requestId); @@ -133,6 +169,8 @@ export async function detectInterruptedRequests(options: { actionName: entry.actionName, sessionId: entry.sessionId }); + + await wakeSessionOf(requestRecord, now); } // A run whose record says `heartbeatsUntilFinalized` beats from its diff --git a/packages/engine/src/execution/stale-request-sweeper.ts b/packages/engine/src/execution/stale-request-sweeper.ts index 97b8060b20..764d007f48 100644 --- a/packages/engine/src/execution/stale-request-sweeper.ts +++ b/packages/engine/src/execution/stale-request-sweeper.ts @@ -16,7 +16,7 @@ * executor's heartbeat always lands inside it. */ import { detectInterruptedRequests } from "./request-recovery"; -import type { StoreRegistry } from "../stores/types"; +import type { RequestRecord, StoreRegistry } from "../stores/types"; import { DEFAULT_RUNTIME_LOGGER, logRuntimeEvent, @@ -52,6 +52,12 @@ export type CreateStaleRequestSweeperOptions = { * Default: 10000 (10s) — matches `runAction`'s default. */ registryHeartbeatMs?: number; + /** + * The internal entries of the flow a request ran on, so a request this sweep + * marks interrupted brings its session's wakes forward (FIX-2166). See + * `detectInterruptedRequests`. + */ + wakeEntriesOf?: (record: RequestRecord) => readonly string[]; logger?: RuntimeLogger; }; @@ -82,6 +88,7 @@ export function createStaleRequestSweeper( staleThresholdMs = DEFAULT_THRESHOLD_MS, queuedGraceMs, registryHeartbeatMs = DEFAULT_REGISTRY_HEARTBEAT_MS, + wakeEntriesOf, logger = DEFAULT_RUNTIME_LOGGER } = options; @@ -114,7 +121,13 @@ export function createStaleRequestSweeper( let inFlight: Promise | undefined; const tick = (): void => { if (disposed || inFlight !== undefined) return; - inFlight = detectInterruptedRequests({ stores, staleThresholdMs, queuedGraceMs, logger }) + inFlight = detectInterruptedRequests({ + stores, + staleThresholdMs, + queuedGraceMs, + ...(wakeEntriesOf !== undefined ? { wakeEntriesOf } : {}), + logger + }) .catch((err) => { logRuntimeEvent( logger, diff --git a/packages/engine/src/flowstate/createFlowState.ts b/packages/engine/src/flowstate/createFlowState.ts index fb7c9f80ca..c39e304fbc 100644 --- a/packages/engine/src/flowstate/createFlowState.ts +++ b/packages/engine/src/flowstate/createFlowState.ts @@ -31,7 +31,7 @@ import { type RuntimeLoggerLevel } from "../execution/logging"; import { abortRequest } from "../execution/abort-registry"; -import { detectInterruptedRequests } from "../execution/request-recovery"; +import { detectInterruptedRequests, internalEntriesFrom } from "../execution/request-recovery"; import { resolveDurabilityProvider } from "../durability/default-durability"; import { FlowStateConfigError, FlowStateDisposedError } from "../errors/flow-error"; import type { CapabilitySlot, StoreAdapter, StoresConfig } from "../stores/store-adapter"; @@ -842,6 +842,8 @@ class InternalFlowState stores, staleThresholdMs, queuedGraceMs, + // A request reaped here owes its session's passes now (FIX-2166). + wakeEntriesOf: internalEntriesFrom(this.#registry), // Resolved per call rather than captured, for the same reason // `#logShutdown` does it: the CLI installs its logger onto the resolved // config AFTER `getRuntime()` returns, and this sweep is already running diff --git a/packages/engine/src/routes/createFlowApiRouter.ts b/packages/engine/src/routes/createFlowApiRouter.ts index 3c4df2e437..8b504cf48e 100644 --- a/packages/engine/src/routes/createFlowApiRouter.ts +++ b/packages/engine/src/routes/createFlowApiRouter.ts @@ -32,6 +32,7 @@ import { } from "../transports/http/createHttpTransportAdapter"; import { TransportRouteCollisionError } from "../transports/errors"; import { createStaleRequestSweeper } from "../execution/stale-request-sweeper"; +import { internalEntriesFrom } from "../execution/request-recovery"; import { drainRequestEnds } from "../execution/request-end-hooks"; import { assertPublicReentrySources } from "./public-reentry"; import { createDurabilitySweeper } from "../durability/durability-sweeper"; @@ -499,7 +500,8 @@ export function createFlowApiRouter(options: CreateFlowApiRouterOptions): FlowAp stores: handlers.host.stores, intervalMs: staleSweepIntervalMs, staleThresholdMs: staleSweepThresholdMs, - queuedGraceMs + queuedGraceMs, + wakeEntriesOf: internalEntriesFrom(options.registry) }); // Server-internal durability sweeper (FIX-141): enforces suspension expiry, diff --git a/packages/engine/src/routes/http-handlers.ts b/packages/engine/src/routes/http-handlers.ts index 2ea8c6de7a..5697e955c2 100644 --- a/packages/engine/src/routes/http-handlers.ts +++ b/packages/engine/src/routes/http-handlers.ts @@ -12,7 +12,7 @@ import type { FlowRegistry } from "../registry/flow-registry"; import type { RuntimeConfig } from "../runtime-config"; import { createInMemoryStores } from "../stores"; import type { StoreRegistry } from "../stores/types"; -import { detectInterruptedRequests } from "../execution/request-recovery"; +import { detectInterruptedRequests, internalEntriesFrom } from "../execution/request-recovery"; import { parseFlowRoute, type ParsedFlowRoute @@ -384,7 +384,9 @@ export function createFlowRouteHandlers(options: CreateFlowRouteHandlersOptions) options.staleThresholdMs ?? runtimeConfig.requestHost?.staleThresholdMs, // Same grace the periodic sweeper runs with, so a restart cannot reap a // queued row the running server would have left alone. - queuedGraceMs: runtimeConfig.queuedGraceMs + queuedGraceMs: runtimeConfig.queuedGraceMs, + // A request reaped here owes its session's passes now (FIX-2166). + wakeEntriesOf: internalEntriesFrom(options.registry) }).then((interrupted) => { if (interrupted.length > 0) { console.warn( diff --git a/packages/engine/test/context/liveness-gate.test.ts b/packages/engine/test/context/liveness-gate.test.ts index 0fb875ba27..bdd46a628b 100644 --- a/packages/engine/test/context/liveness-gate.test.ts +++ b/packages/engine/test/context/liveness-gate.test.ts @@ -31,7 +31,7 @@ * cadence is tuned. */ import { describe, it, expect } from "vitest"; -import { evaluateLivenessGate } from "../../src/context/liveness-gate"; +import { evaluateLivenessGate, heartbeatStaleThreshold } from "../../src/context/liveness-gate"; import type { ActiveRequestRegistry } from "../../src/stores/types"; const shared = { sharedAcrossProcesses: true } as ActiveRequestRegistry; @@ -120,3 +120,17 @@ describe("liveness gate", () => { } }); }); + +describe("the stale threshold a request's heartbeat is read against (FIX-2166)", () => { + // `waits.running` reads a stale heartbeat as a stopped request only where a + // healthy one can't look stale: heartbeats on, at least twice per threshold. + it("is the sweep's threshold where heartbeats beat often enough, with the same defaults", () => { + expect(heartbeatStaleThreshold({ heartbeatIntervalMs: undefined, staleThresholdMs: undefined })).toBe(60_000); + expect(heartbeatStaleThreshold({ heartbeatIntervalMs: 5_000, staleThresholdMs: 10_000 })).toBe(10_000); + }); + + it("is absent with heartbeats off or too slow for the threshold, so no healthy request reads as stopped", () => { + expect(heartbeatStaleThreshold({ heartbeatIntervalMs: 0, staleThresholdMs: 60_000 })).toBeUndefined(); + expect(heartbeatStaleThreshold({ heartbeatIntervalMs: 40_000, staleThresholdMs: 60_000 })).toBeUndefined(); + }); +}); diff --git a/packages/engine/test/request-recovery.test.ts b/packages/engine/test/request-recovery.test.ts index 067fdb06eb..453211de80 100644 --- a/packages/engine/test/request-recovery.test.ts +++ b/packages/engine/test/request-recovery.test.ts @@ -428,4 +428,100 @@ describe("detectInterruptedRequests", () => { const record = await stores.request.get("req_fresh"); expect(record!.status).toBe("in_progress"); }); + + describe("a request it marks interrupted owes its session's passes now (FIX-2166)", () => { + // A pass in that session may hold something for the request while it runs + // (a task's notice, FIX-2123). The request will never end on its own now, + // so what was held is owed now, not at the pass's backoff. + const LATER = Date.now() + 300_000; + const wake = (sessionId: string, entry: string, dueAt = LATER) => ({ + sessionId, + entry, + flowKind: "chat", + userId: "user_1", + orgId: DEFAULT_ORG_ID, + dueAt, + landBy: dueAt + }); + const crashed = async (requestId: string, sessionId: string, lastHeartbeatAt = Date.now() - 60_000) => { + await stores.activeRequests.register({ + orgId: DEFAULT_ORG_ID, + requestId, + flowKind: "chat", + actionName: "run", + source: "http", + userId: "user_1", + sessionId, + startedAt: lastHeartbeatAt, + lastHeartbeatAt + }); + await stores.request.set(requestId, makeRequestRecord(requestId, { sessionId }), "any"); + }; + const dueOf = async (sessionId: string, entry: string) => (await stores.wakes!.get({ sessionId, entry }))?.dueAt; + + it("brings each wake its session keeps forward to now, and no other session's", async () => { + await crashed("req_crashed", "s_filer"); + await stores.wakes!.owe(wake("s_filer", "onTaskSettled")); + await stores.wakes!.owe(wake("s_filer", "dispatch")); + await stores.wakes!.owe(wake("s_other", "onTaskSettled")); + const before = Date.now(); + + await detectInterruptedRequests({ stores, staleThresholdMs: 30_000, wakeEntriesOf: () => ["onTaskSettled", "dispatch", "neverOwed"] }); + + expect((await stores.request.get("req_crashed"))?.status).toBe("interrupted"); + expect(await dueOf("s_filer", "onTaskSettled")).toBeGreaterThanOrEqual(before); + expect(await dueOf("s_filer", "onTaskSettled")).toBeLessThanOrEqual(Date.now()); + expect(await dueOf("s_filer", "dispatch")).toBeLessThanOrEqual(Date.now()); + // Nothing is written where nothing was owed. + expect(await stores.wakes!.get({ sessionId: "s_filer", entry: "neverOwed" })).toBeNull(); + expect(await dueOf("s_other", "onTaskSettled")).toBe(LATER); + }); + + it("bumps the wake's version, so a pass that read it before settles stale and leaves it due now", async () => { + await crashed("req_racing", "s_racing"); + await stores.wakes!.owe(wake("s_racing", "onTaskSettled")); + // A pass in flight read the wake while the request still looked alive. + const read = (await stores.wakes!.get({ sessionId: "s_racing", entry: "onTaskSettled" }))!; + + await detectInterruptedRequests({ stores, staleThresholdMs: 30_000, wakeEntriesOf: () => ["onTaskSettled"] }); + + // That pass now settles to its backoff, at the version it read. + expect(await stores.wakes!.move({ sessionId: "s_racing", entry: "onTaskSettled" }, read.version, LATER)).toBe(false); + expect(await dueOf("s_racing", "onTaskSettled")).toBeLessThanOrEqual(Date.now()); + }); + + it("leaves the wakes of a request it doesn't mark, and every wake when its caller names no entries", async () => { + // Still beating: maybe on a live peer, so nothing it holds is owed yet. + await crashed("req_beating", "s_beating", Date.now()); + await stores.wakes!.owe(wake("s_beating", "onTaskSettled")); + await crashed("req_unnamed", "s_unnamed"); + await stores.wakes!.owe(wake("s_unnamed", "onTaskSettled")); + + await detectInterruptedRequests({ stores, staleThresholdMs: 30_000, wakeEntriesOf: (record) => (record.sessionId === "s_beating" ? ["onTaskSettled"] : []) }); + + expect(await dueOf("s_beating", "onTaskSettled")).toBe(LATER); + expect((await stores.request.get("req_unnamed"))?.status).toBe("interrupted"); + expect(await dueOf("s_unnamed", "onTaskSettled")).toBe(LATER); + }); + + it("still marks the request when the wake store fails", async () => { + await crashed("req_blip", "s_blip"); + const warned: string[] = []; + const failing = Object.assign(Object.create(stores.wakes!), { + get: async () => Promise.reject(new Error("store unavailable")) + }) as NonNullable; + + const interrupted = await detectInterruptedRequests({ + stores: { ...stores, wakes: failing }, + staleThresholdMs: 30_000, + wakeEntriesOf: () => ["onTaskSettled"], + logger: { warn: (message) => warned.push(message) } + }); + + expect(warned.some((message) => message.includes("wakes weren't brought forward"))).toBe(true); + + expect(interrupted.map((info) => info.entry.requestId)).toEqual(["req_blip"]); + expect((await stores.request.get("req_blip"))?.status).toBe("interrupted"); + }); + }); }); diff --git a/packages/engine/test/wait-chain.test.ts b/packages/engine/test/wait-chain.test.ts index 193385a47e..d439c4c7cc 100644 --- a/packages/engine/test/wait-chain.test.ts +++ b/packages/engine/test/wait-chain.test.ts @@ -207,6 +207,74 @@ describe("whether a request has stopped running its own work", () => { unreadable: false }); }); + + describe("a request whose process died before the sweep marked it (FIX-2166)", () => { + // Its record still reads `in_progress`; only its heartbeat stopped. + const NOW = 1_000_000; + const THRESHOLD = 60_000; + const inProgress = (id: string) => ({ id, status: "in_progress", sessionId: "s" }); + const entries: Record = { + beating: { lastHeartbeatAt: NOW - 10_000 }, + // Last beat just inside the threshold: a request on a live peer, between beats. + atThreshold: { lastHeartbeatAt: NOW - THRESHOLD }, + stopped: { lastHeartbeatAt: NOW - THRESHOLD - 1 }, + // Queued for an external worker: nothing beats for it yet, and it is not dead. + queued: { lastHeartbeatAt: NOW - 10 * THRESHOLD, queuedAt: NOW - 10 * THRESHOLD }, + // A custom registry that records no heartbeat: no proof of anything. + noHeartbeat: {} as { lastHeartbeatAt: number } + }; + const waitsWith = (heartbeat?: { staleThresholdMs: number }) => + createRequestWaits({ + provider: { ...provider([]), loadSuspension: async () => null as never }, + deadlines: { defaultMs: 1, minMs: 1, maxMs: 1 }, + userId: "u", + requestId: "r", + own: undefined, + sessionId: "s", + readRequest: async (id) => inProgress(id) as never, + ...(heartbeat !== undefined + ? { heartbeat: { readEntry: async (id) => entries[id], staleThresholdMs: heartbeat.staleThresholdMs, now: () => NOW } } + : {}) + }); + + it("reads it as stopped once its heartbeat is stale past the sweep's threshold, and no sooner", async () => { + // A notice held for it is sent then, rather than when the sweep gets to it. + expect(await waitsWith({ staleThresholdMs: THRESHOLD }).running!(["beating", "atThreshold", "stopped", "queued", "noHeartbeat", "unregistered"])).toEqual({ + beating: true, + atThreshold: true, + stopped: false, + queued: true, + noHeartbeat: true, + // No entry is no proof its process is gone: the record answers. + unregistered: true + }); + }); + + it("reads the record alone when the registry can't be read: a lookup that fails is no proof its process is gone", async () => { + // Read as stopped, a live request would have its notice sent and its + // lapsed slots released while it still runs. + const blip = createRequestWaits({ + provider: { ...provider([]), loadSuspension: async () => null as never }, + deadlines: { defaultMs: 1, minMs: 1, maxMs: 1 }, + userId: "u", + requestId: "r", + own: undefined, + sessionId: "s", + readRequest: async (id) => inProgress(id) as never, + heartbeat: { + readEntry: async () => Promise.reject(new Error("registry unavailable")), + staleThresholdMs: THRESHOLD, + now: () => NOW + } + }); + expect(await blip.running!(["live"])).toEqual({ live: true }); + }); + + it("reads the record alone where heartbeats can't say: a stale entry there is a healthy request", async () => { + // A flow with heartbeats off never beats, so every entry of it looks stale. + expect(await waitsWith().running!(["stopped"])).toEqual({ stopped: true }); + }); + }); }); describe("the stamp a waited dispatch leaves on its child", () => { diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md index c533173681..7338aa8f43 100644 --- a/packages/orchestration/README.md +++ b/packages/orchestration/README.md @@ -510,10 +510,14 @@ sent again. The entry queues on the session unless `concurrency` says otherwise. A notice waits for the request that filed its task while that request is running, not while it is parked on a person or a wait. The entry holds the notice back the way `defer` does (below). The request sends what it held as it ends; once it parks, the session's pass runs about two seconds -later and sends it then, past its resend backoff. A wait in the filing request therefore reads the ending -first. Before it runs `block`, the entry takes the notice in one conditional write (the board's -optional `takeNotice`, which both built-in backings implement), so a wait that reads the task and -the notice never both deliver the same ending. A notice whose `block` threw stays owed, and the +later and sends it then, past its resend backoff. A wait in the filing request therefore reads the +ending first. A filing request whose server died mid-run holds it only until the engine can tell: +once its heartbeat has been silent past the stale-request threshold, a pass sends it, and the +stale-request sweep that marks the request interrupted brings the session's pass forward to then. +With the defaults that is within about 90 seconds of the request's last heartbeat. Before it runs +`block`, the entry takes the notice in one conditional write (the board's optional `takeNotice`, +which both built-in backings implement), so a wait that reads the task and the notice never both +deliver the same ending. A notice whose `block` threw stays owed, and the next copy takes it over; a copy that finds the notice taken by a request that hasn't ended, running or parked, runs nothing. Holding needs a host that records which requests are running (durable execution, `requestHost.waits.running`); anywhere else a notice is admitted at once. diff --git a/packages/orchestration/test/tasks/notice-after-crash.test.ts b/packages/orchestration/test/tasks/notice-after-crash.test.ts new file mode 100644 index 0000000000..b7201f910b --- /dev/null +++ b/packages/orchestration/test/tasks/notice-after-crash.test.ts @@ -0,0 +1,211 @@ +/** + * A task's notice held for the request that filed it reaches its filer after + * that request's process dies mid-run (FIX-2166). + * + * FIX-2123 holds a notice while its filing request runs, so a wait in that + * request reads the ending first. A request whose process died still reads + * `in_progress` until the stale-request sweep marks it interrupted, and + * nothing then sent the held notice before the pass's next resend, its + * backoff (up to 5 minutes) later. Now the request reads as stopped once its + * heartbeat is stale past the sweep's threshold, and the sweep that marks it + * interrupted brings the filer's pass forward, so the notice is heard within + * the threshold and one sweep tick of the request's last heartbeat. + * + * The crash is a seam: the first engine runs the filing turn and is then + * abandoned, its model call never answering and its heartbeat never beating + * again, with nothing ended or deregistered; a second engine starts on the + * same stores. Short thresholds stand in for the defaults' minutes. + * + * Two cases, one per way the notice is freed: the sweep marking the request + * interrupted and bringing the pass forward; and, with sweeping off, the pass + * itself reading the request's stale heartbeat as stopped. + * + * Each runs on the in-memory stores and on SQLite. + */ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { DEFAULT_ORG_ID } from "@flow-state-dev/core"; +import type { FlowInstance } from "@flow-state-dev/core/types"; +import { createFlowState, disposeFlowApiRouter, inMemoryStores, runAction, type StoreAdapter } from "@flow-state-dev/engine"; +import { sqliteStores } from "@flow-state-dev/store-sqlite"; +import type { TaskSettledInput } from "../../src"; +import { act, fileCall, type Row, SESSION, stepModel, USER, waitFlow } from "./wait-fixture"; + +const dirs: string[] = []; +afterEach(async () => { + for (const dir of dirs.splice(0)) await rm(dir, { recursive: true, force: true }); +}); + +/** Each case opens the same stores twice: once per engine, as a restarted server does. */ +const STORES: ReadonlyArray Promise<() => Promise>]> = [ + [ + "in-memory", + async () => { + const shared = inMemoryStores(); + return async () => shared; + } + ], + [ + "sqlite", + async () => { + const dir = await mkdtemp(join(tmpdir(), "fsd-notice-after-crash-")); + dirs.push(dir); + return async () => sqliteStores({ filename: join(dir, "store.db") }); + } + ] +]; + +/** The sweep's stale threshold and cadence here: the defaults are 60 s and 30 s. */ +const THRESHOLD_MS = 7_000; +const SWEEP_MS = 500; +/** + * A wake brought forward fires within the durability sweep's 1 s floor, and + * the pass and the notice's request then run: slack for a loaded machine, + * still well short of the resend that would otherwise send it (14 s here). + */ +const PASS_MS = 3_000; + +const settle = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +async function eventually(read: () => Promise, withinMs: number, what: string): Promise { + const deadline = Date.now() + withinMs; + for (;;) { + const value = await read(); + if (value !== undefined) return value; + if (Date.now() > deadline) throw new Error(`${what}: not within ${withinMs} ms`); + await settle(25); + } +} + +function engine(flow: FlowInstance, primary: StoreAdapter, sweepMs: number) { + return createFlowState({ + flows: { "wait-board": flow }, + stores: { default: { primary } }, + durable: true, + staleSweepThresholdMs: THRESHOLD_MS, + staleSweepIntervalMs: sweepMs + }); +} + +/** + * A filing request left mid-run by a dead server, its task's notice held for + * it, and a second server started on the same stores with the stale sweep at + * `sweepMs` (0: off). + */ +async function crashAndRestart(open: () => Promise, sweepMs: number) { + const notices: TaskSettledInput[] = []; + const heardAt: number[] = []; + const heard = { heard: notices, heardGate: async () => void heardAt.push(Date.now()) }; + + // The first server: the turn files a task that ends at once, then its + // model call never answers. Its heartbeat beats once an hour, which here + // is never: the request is left as a killed process leaves it. + // The call answers only the teardown's abort, so the run ends before the + // next test file: a run left hanging holds this process's request-end + // drain, which every later host's shutdown waits on. + const kill = new AbortController(); + const neverAnswers = (opts: { signal?: AbortSignal }) => + new Promise((_, reject) => (opts.signal ?? kill.signal).addEventListener("abort", () => reject(new Error("killed")), { once: true })); + const { model } = stepModel([fileCall("f1"), neverAnswers as never]); + const first = engine(waitFlow(model, { endsOnFiling: "already done", heartbeatIntervalMs: 3_600_000, ...heard }), await open(), sweepMs); + const firstRuntime = await first.getRuntime(); + const flow = firstRuntime.registry.get("wait-board")!; + const abandoned = runAction({ + signal: kill.signal, + orgId: DEFAULT_ORG_ID, + flow, + actionName: "run", + input: {}, + userId: USER, + sessionId: SESSION, + stores: firstRuntime.stores, + runtimeConfig: firstRuntime.runtimeConfig + }).catch(() => undefined); + + // The notice is turned away for the running filing request, as it is + // while that request lives. + const filing = await eventually( + async () => { + // Read without a router: the first server runs no sweeps, as a server about to die runs none we can count on. + const listed = (await act(first, flow, "list", {}, { sweeper: false })).output as Row[]; + const meta = listed[0]?.metadata ?? {}; + const deferred = Object.entries(meta).find(([key]) => key.startsWith("noticeDeferred:"))?.[1] as { whileRunning?: string } | undefined; + return deferred?.whileRunning; + }, + 10_000, + "the notice was never held for its filing request" + ); + const entry = await firstRuntime.stores.activeRequests.get(filing); + expect(entry).toBeDefined(); + const lastBeat = entry!.lastHeartbeatAt; + expect(notices).toEqual([]); + + // The restart: a second server on the same stores, nothing sent to it. + // Its flow beats as a healthy one does, so a stale entry reads as stopped. + const { model: idle } = stepModel([]); + const second = engine(waitFlow(idle, { heartbeatIntervalMs: 1_000, ...heard }), await open(), sweepMs); + const router = await second.getRouter(); + return { + notices, + statusOf: async () => (await (await second.getRuntime()).stores.request.get(filing))?.status, + /** Resolves once the notice is heard, with how long after the request's last heartbeat. */ + heard: async (withinMs: number) => { + await eventually(async () => (notices.length > 0 ? true : undefined), withinMs, "the notice was never heard"); + return heardAt[0]! - lastBeat; + }, + dispose: async () => { + kill.abort(); + const ended = await Promise.race([abandoned.then(() => true), settle(5_000).then(() => false)]); + expect(ended, "the abandoned run ended at the teardown's abort").toBe(true); + await disposeFlowApiRouter(router, { drainMs: 100 }); + await second.dispose(); + await first.dispose().catch(() => undefined); + } + }; +} + +describe.each(STORES)("a notice held for a filing request whose process died (%s)", { timeout: 60_000 }, (_name, open) => { + it("is heard within the stale threshold and one sweep tick of that request's last heartbeat, not at its resend's backoff", async () => { + const run = await crashAndRestart(await open(), SWEEP_MS); + try { + const sinceBeat = await run.heard(THRESHOLD_MS + 15_000); + expect(run.notices).toEqual([expect.objectContaining({ ending: "completed", output: "already done" })]); + // Not while the request could still be alive on another server: only + // once its heartbeat is stale past the threshold the sweep reaps at. + expect(sinceBeat).toBeGreaterThanOrEqual(THRESHOLD_MS); + // And within one sweep tick and one pass of that, not at the backoff. + expect(sinceBeat).toBeLessThanOrEqual(THRESHOLD_MS + SWEEP_MS + PASS_MS); + // A pass that read the stale heartbeat may have sent it just before the + // sweep's tick: the sweep marks the request all the same. + await eventually(async () => ((await run.statusOf()) === "interrupted" ? true : undefined), SWEEP_MS + 2_000, "the sweep never marked the request"); + + await settle(2_500); + expect(run.notices).toHaveLength(1); + } finally { + await run.dispose(); + } + }); + + it("with the stale sweep off, is heard at the first resend after that request's heartbeat goes stale, though nothing marks it", async () => { + // Nothing marks the request interrupted here, so only the pass's own read + // of its heartbeat frees the notice. Its resends run about 2, 6 and 14 s + // after the ending; the threshold falls between the second and the third. + const run = await crashAndRestart(await open(), 0); + try { + const sinceBeat = await run.heard(THRESHOLD_MS + 15_000); + expect(run.notices).toEqual([expect.objectContaining({ ending: "completed", output: "already done" })]); + expect(sinceBeat).toBeGreaterThanOrEqual(THRESHOLD_MS); + // At most one backoff step (8 s here) past the threshold, and its pass. + expect(sinceBeat).toBeLessThanOrEqual(THRESHOLD_MS + 8_000 + PASS_MS); + expect(await run.statusOf()).toBe("in_progress"); + + await settle(2_500); + expect(run.notices).toHaveLength(1); + } finally { + await run.dispose(); + } + }); +}); + diff --git a/packages/orchestration/test/tasks/wait-fixture.ts b/packages/orchestration/test/tasks/wait-fixture.ts index e6339161f0..6fc9194c5e 100644 --- a/packages/orchestration/test/tasks/wait-fixture.ts +++ b/packages/orchestration/test/tasks/wait-fixture.ts @@ -143,6 +143,8 @@ export function waitFlow( * parked, as a slow hook on a real host does. */ parkTakesMs?: number; + /** The flow's request heartbeat (`request.heartbeatIntervalMs`); the engine's default when absent. */ + heartbeatIntervalMs?: number; } = {} ): FlowInstance { let cancelFailed = false; @@ -342,6 +344,7 @@ export function waitFlow( // Its rows' endings are delivered to their filer, and each row names it // (FIX-2078): only the session that filed a task may wait on it. resources: { [BOARD]: defineTaskCollection({ id: BOARD, scope: options.scope ?? "session", delivery: {} }) }, + ...(options.heartbeatIntervalMs !== undefined ? { request: { heartbeatIntervalMs: options.heartbeatIntervalMs } } : {}), actions: { run: { block: sequencer({ name: "turn" }).step(