diff --git a/.changeset/one-turn-of-a-task-at-a-time.md b/.changeset/one-turn-of-a-task-at-a-time.md new file mode 100644 index 0000000000..2ea6b7ca25 --- /dev/null +++ b/.changeset/one-turn-of-a-task-at-a-time.md @@ -0,0 +1,6 @@ +--- +"@flow-state-dev/orchestration": patch +"@flow-state-dev/workforce": patch +--- + +On a Workforce conversation's board, a task runs one turn at a time in its session: while one of its turns waits on a step it filed for itself in a session of its own, a notice's turn and a steer's turn wait until it answers, instead of planning beside it and stalling the task. `taskSettledEntry` takes `defer`, and `resendNowPatch` is exported (FIX-2113). diff --git a/.changeset/steer-turn-shows-its-steers.md b/.changeset/steer-turn-shows-its-steers.md new file mode 100644 index 0000000000..6c63b0e388 --- /dev/null +++ b/.changeset/steer-turn-shows-its-steers.md @@ -0,0 +1,5 @@ +--- +"@flow-state-dev/workforce": patch +--- + +The turn a steer starts while a task waits on work it filed now shows the steer in the task's session as a line from its sender, so the turn that answers after that work ends reads it too (FIX-2113). diff --git a/apps/docs/docs/orchestration/task-board.md b/apps/docs/docs/orchestration/task-board.md index 4992ea755b..f627e58071 100644 --- a/apps/docs/docs/orchestration/task-board.md +++ b/apps/docs/docs/orchestration/task-board.md @@ -91,14 +91,14 @@ Every rule is checked inside the write, so a task that starts between your read - **Filed, not started yet.** The first attempt's brief carries every message, after the goal. - **Running.** The worker takes it before its next model call, in the same turn. A coding run on a [harness manager](./harness-manager.md#steering-a-run) takes it at its next step. -- **Waiting on tasks it filed.** A turn starts now, with the message, while those tasks keep running. +- **Waiting on tasks it filed.** A turn starts with the message while those tasks keep running. On a Workforce conversation's board, a session runs one turn of a task at a time. If another of the task's turns is waiting on a step it filed for itself in a session of its own, the message's turn waits until that turn answers. So does the turn that a finished task wakes. - **After the worker's last model call**, even while the task is being settled. The task gets one more turn, with the message, before it settles. Its output follows the usual rule: the latest checkpoint, or else that turn's answer. The last case needs a turn to run. A Workforce conversation's board has one. For a board you wire yourself, name the internal entry that runs it in `taskLedgers({ endRules: { steerTurn: { entry } } })`. Without one, the task settles and the message stays on it, untaken: on such a board, a message that arrives after the last model call may not be acted on. The filer's model sends a message with the `updateTask` tool, and a person with the board's `updateTask` action; the task records which. A message is sent alone, not with an edit or a hold, so a call never lands half of what it asked. A task takes at most 20 messages. Past that, cancel it and file it again with everything it needs, or file a follow-up once it ends. -`getTask` lists a task's messages as `steers`, oldest first, each with who sent it and whether its work has read it (`taken`). A message the work reads mid-run also shows in the task's session as a line from its sender. +`getTask` lists a task's messages as `steers`, oldest first, each with who sent it and whether its work has read it (`taken`). Unless it arrived before the task started, the message also shows in the task's session as a line from its sender, so later turns in that session read it. A finished task takes no message, because its output never changes. To correct it, file a follow-up: it runs in the task's session, and `getTask` on the original shows the follow-up's output as `current`. diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md index 7f1283ef1c..9f4a184f9b 100644 --- a/packages/orchestration/README.md +++ b/packages/orchestration/README.md @@ -496,7 +496,7 @@ export default defineFlow({ ``` **Hearing how a task ended.** The session that filed a row hears each of its endings once, as -a request to its flow's `onTaskSettled` internal entry. `taskSettledEntry({ block, concurrency? })` +a request to its flow's `onTaskSettled` internal entry. `taskSettledEntry({ block, concurrency?, defer? })` builds that entry; `defineFlow` refuses a flow that holds a board that dispatches without one, whichever flow the rows run on. `block` takes `taskSettledInputSchema` / `TaskSettledInput`: `{ boardId, taskId, attempt, ending, output?, error?, question?, cancelled? }`, where `ending` is @@ -507,6 +507,16 @@ the filer waited on (`getTask` with `wait`), nor a cancel the filer's own model task tool in its turn. The block runs once per ending; if it throws, the notice stays owed and is sent again. The entry queues on the session unless `concurrency` says otherwise. +**Holding a notice back.** Give `taskSettledEntry` a `defer(ctx, notice)` when the session can be +busy with something a notice must not run beside, such as a turn that is still under way. When it +answers `true`, the entry runs nothing and acknowledges nothing: the notice stays owed, and the +session's pass (below) sends it again. Each resend waits longer than the last (2 s, doubling, at most +5 minutes): that wait is the notice's *resend backoff*. A notice held back is not a failed delivery: +its age, which ends it as undeliverable past `maxDebtAgeMs`, counts from the last time it was held +back. Once the session is free, apply `resendNowPatch(row)` to each row that still owes a notice, +with `ref.patchMetadata(row.id, resendNowPatch(row))`, and run the pass: the patch drops the resend +backoff, so the pass sends the notice at once rather than at its next resend. + A notice that couldn't be sent when the row ended (the process died, the send failed) is sent about two seconds later, or as soon as the server is running again if it was down then: on a host with durable execution and a store that keeps wakes, the durability sweeper runs `onTaskSettled` in the filing session with an empty input, which sends diff --git a/packages/orchestration/src/tasks/index.ts b/packages/orchestration/src/tasks/index.ts index e12e0be347..9fe634edce 100644 --- a/packages/orchestration/src/tasks/index.ts +++ b/packages/orchestration/src/tasks/index.ts @@ -206,6 +206,7 @@ export { type NoticeDecision, NOTICE_DUE_MS, noticeDueAt, + resendNowPatch, cancelledDependents, type NoticeRecording, } from "./notice/task-notice"; diff --git a/packages/orchestration/src/tasks/notice/pass.ts b/packages/orchestration/src/tasks/notice/pass.ts index 55cede73f3..1ed943d84b 100644 --- a/packages/orchestration/src/tasks/notice/pass.ts +++ b/packages/orchestration/src/tasks/notice/pass.ts @@ -123,8 +123,10 @@ export async function payBoardDebts( for (const notice of owedNotices(row, ref.collectionId, rows, options.filerOf)) { const times = noticeTimes(row, notice); if (times === undefined) continue; - if (tooOld(times.due)) { - const detail = `it was still owed ${Math.round((now - times.due) / 1000)} s past its due time`; + // Its age counts from its due time, or from the last time a busy session + // turned it away (FIX-2113), which is no failed delivery. + if (tooOld(times.since)) { + const detail = `it was still owed ${Math.round((now - times.since) / 1000)} s past its due time`; log("a notice past the maximum debt age was settled undeliverable", { taskId: row.id, ending: notice.ending }); await ref.patchMetadata(row.id, { ...clearNotice(notice), ...undeliverableRecord(notice, detail, now) }); await options.onUndeliverable?.(ctx, row, notice, detail); diff --git a/packages/orchestration/src/tasks/notice/task-notice.ts b/packages/orchestration/src/tasks/notice/task-notice.ts index 25aacb9ee0..5ea143236c 100644 --- a/packages/orchestration/src/tasks/notice/task-notice.ts +++ b/packages/orchestration/src/tasks/notice/task-notice.ts @@ -119,6 +119,15 @@ const OWED_PREFIX = "noticeOwed:"; */ const RESENT_PREFIX = "noticeResent:"; +/** + * Where a busy session records that it turned an owed notice away (FIX-2113: + * `taskSettledEntry`'s `defer`), and when. That is no failed delivery, so the + * notice's age, which ends it at the maximum debt age, counts from then. Kept + * beside the marker, never in it, as a resend record is, so the write can't + * bring back a marker its entry cleared meanwhile. + */ +const DEFERRED_PREFIX = "noticeDeferred:"; + /** The longest a pass waits between two resends of one notice: 5 minutes. */ const NOTICE_RESEND_CAP_MS = 300_000; @@ -248,6 +257,23 @@ export function releasedHeldNotices(row: Task, dueAt: number): Record { + const patch: Record = {}; + for (const [key, value] of Object.entries(row.metadata ?? {})) { + if (!isSendable(key, value)) continue; + const resent = `${RESENT_PREFIX}${value.attempt}:${value.ending}`; + if ((row.metadata ?? {})[resent] != null) patch[resent] = null; + } + return patch; +} + /** Whether `row` holds an ending for the request whose wait on it lapsed (FIX-2082). */ export function holdsAnEnding(row: Task): boolean { for (const [key, value] of Object.entries(row.metadata ?? {})) { @@ -341,14 +367,31 @@ function nextSendAt(row: Task, marker: OwedMarker): number { } /** - * When `notice` first fell due, and when it is next sent (see `nextSendAt`). - * `undefined` when `row` no longer owes it. Its age counts from `due`. + * When `notice` first fell due, when it is next sent (see `nextSendAt`), and + * when its age counts from: its due time, or the last time a busy session + * turned it away, whichever is later. `undefined` when `row` no longer owes it. */ -export function noticeTimes(row: Task, notice: Pick): { due: number; next: number } | undefined { +export function noticeTimes( + row: Task, + notice: Pick +): { due: number; next: number; since: number } | undefined { const key = markerKey(notice.attempt, notice.ending); const value = (row.metadata ?? {})[key]; if (!isSendable(key, value)) return undefined; - return { due: value.due ?? 0, next: nextSendAt(row, value) }; + const due = value.due ?? 0; + const deferred = (row.metadata ?? {})[`${DEFERRED_PREFIX}${notice.attempt}:${notice.ending}`] as { at?: unknown } | null | undefined; + const deferredAt = deferred != null && typeof deferred.at === "number" ? deferred.at : 0; + return { due, next: nextSendAt(row, value), since: Math.max(due, deferredAt) }; +} + +/** + * The metadata patch that records a busy session turning `notice` away at + * `now` (FIX-2113): a record beside its marker, which it leaves as it is. + * `{}` when `row` no longer owes it. + */ +export function deferredNoticePatch(row: Task, notice: Pick, now: number): Record { + if (!isSendable(markerKey(notice.attempt, notice.ending), (row.metadata ?? {})[markerKey(notice.attempt, notice.ending)])) return {}; + return { [`${DEFERRED_PREFIX}${notice.attempt}:${notice.ending}`]: { at: now } }; } /** The metadata patch that records one more resend of `notice` at `now`. */ @@ -490,7 +533,11 @@ export function withoutNoticeMarkers( if (metadata === undefined) return undefined; return Object.fromEntries( Object.entries(metadata).filter( - ([key]) => !key.startsWith(OWED_PREFIX) && !key.startsWith(UNDELIVERABLE_PREFIX) && !key.startsWith(RESENT_PREFIX) + ([key]) => + !key.startsWith(OWED_PREFIX) && + !key.startsWith(UNDELIVERABLE_PREFIX) && + !key.startsWith(RESENT_PREFIX) && + !key.startsWith(DEFERRED_PREFIX) ) ); } diff --git a/packages/orchestration/src/tasks/notice/task-settled-entry.ts b/packages/orchestration/src/tasks/notice/task-settled-entry.ts index 3feb597e7a..d615c148b7 100644 --- a/packages/orchestration/src/tasks/notice/task-settled-entry.ts +++ b/packages/orchestration/src/tasks/notice/task-settled-entry.ts @@ -25,7 +25,7 @@ import { handler, sequencer } from "@flow-state-dev/core"; import type { BlockContext, BlockDefinition, ConcurrencyConfig } from "@flow-state-dev/core/types"; import { z } from "zod"; -import { clearNotice, isNoticeOwed, type TaskNotice } from "./task-notice"; +import { clearNotice, deferredNoticePatch, isNoticeOwed, type TaskNotice } from "./task-notice"; import { filerResolver, TASK_SETTLED_ENTRY } from "./delivery"; import { delegatingBoardsOf, payBoardDebts, settlePassWake, type BoardDebts } from "./pass"; @@ -62,6 +62,16 @@ export interface TaskSettledEntryOptions { * one notice run one after the other and the second finds it acknowledged. */ readonly concurrency?: ConcurrencyConfig; + /** + * Whether this notice must wait: `true` runs nothing and acknowledges + * nothing, so the notice stays owed and the filer's pass sends it again + * (with its backoff, at most every 5 minutes). For a session that is busy + * with something the notice must not run beside. Being turned away is no + * failed delivery: the notice's age, which ends it as undeliverable past + * the host's maximum debt age, counts from the last time it was turned + * away. Default: never. + */ + readonly defer?: (ctx: BlockContext, notice: TaskSettledInput) => Promise; } /** The running session's board a notice names, at this session's partition. */ @@ -103,6 +113,12 @@ export function taskSettledEntry(options: TaskSettledEntryOptions) { const row = board?.ref.get(notice.taskId); // A notice its row doesn't owe was delivered already, or never owed. if (row === undefined || !isNoticeOwed(row, notice)) return { owed: false }; + // Not now: left owed, for the pass to send again, with the time it was + // turned away, so its age (which ends it as undeliverable) counts from then. + if (options.defer !== undefined && (await options.defer(ctx, notice))) { + await board!.ref.patchMetadata(row.id, deferredNoticePatch(row, notice, board!.ref.now())); + return { owed: false }; + } return { owed: true, notice }; } }); diff --git a/packages/orchestration/test/tasks/notice-defer.test.ts b/packages/orchestration/test/tasks/notice-defer.test.ts new file mode 100644 index 0000000000..a5661fe82d --- /dev/null +++ b/packages/orchestration/test/tasks/notice-defer.test.ts @@ -0,0 +1,116 @@ +/** + * `taskSettledEntry`'s `defer` (FIX-2113), on the real engine: a notice its + * session turns away runs nothing and stays owed, stamped with when it was + * turned away (beside its marker), so its age (which ends it as undeliverable) + * counts from then. + * One it doesn't turn away runs and is acknowledged. + */ +import { DEFAULT_ORG_ID, defineFlow, handler } from "@flow-state-dev/core"; +import type { BlockContext } from "@flow-state-dev/core/types"; +import { createFlowState, inMemoryStores, runAction } from "@flow-state-dev/engine"; +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { taskSettledEntry, type TaskSettledInput } from "../../src"; +import { defineTaskCollection, getOrCreateTaskCollection, resolveResourceCollection, type TaskCollectionRef } from "../../src/tasks"; + +const BOARD = "defer-board"; +const USER = "u_filer"; +const SESSION = "s_filer"; + +async function boardOf(ctx: BlockContext): Promise { + const collection = resolveResourceCollection(ctx, BOARD)!; + return getOrCreateTaskCollection({ ctx, backing: "resource", collectionId: BOARD, collection }); +} + +function host(defer: () => boolean) { + const heard: TaskSettledInput[] = []; + const seed = handler({ + name: "seed", + inputSchema: z.object({ id: z.string() }), + outputSchema: z.any(), + execute: async ({ id }, ctx) => { + const board = await boardOf(ctx); + await board.addTask({ id, goal: "Find the fact" }); + // The ending owes this session its notice, due now. + await board.patchMetadata(id, { "noticeOwed:1:completed": { attempt: 1, ending: "completed", due: Date.now() } }); + return {}; + } + }); + const read = handler({ + name: "read", + inputSchema: z.object({ id: z.string() }), + outputSchema: z.any(), + execute: async ({ id }, ctx) => (await boardOf(ctx)).get(id)?.metadata ?? null + }); + const flow = defineFlow({ + kind: "defer-host", + resources: { [BOARD]: defineTaskCollection({ id: BOARD, scope: "session", delivery: {} }) }, + actions: { seed: { block: seed }, read: { block: read } }, + internal: { + actions: { + onTaskSettled: taskSettledEntry({ + block: handler({ + name: "record-heard", + inputSchema: z.any(), + outputSchema: z.any(), + execute: (notice: TaskSettledInput) => { + heard.push(notice); + return {}; + } + }), + defer: async () => defer() + }) + } + } + })({ id: "defer-host" }); + const state = createFlowState({ flows: { "defer-host": flow }, stores: { default: { primary: inMemoryStores() } } }); + const act = async (actionName: string, input: unknown, internal = false) => { + const runtime = await state.getRuntime(); + const result = await runAction({ + orgId: DEFAULT_ORG_ID, + flow, + actionName, + input, + userId: USER, + sessionId: SESSION, + stores: runtime.stores, + runtimeConfig: runtime.runtimeConfig, + ...(internal ? { source: "internal" } : {}) + } as never); + if ((result as { error?: unknown }).error !== undefined) throw new Error(`${actionName}: ${JSON.stringify((result as { error?: unknown }).error)}`); + return (result as { output?: unknown }).output; + }; + return { heard, act, dispose: () => state.dispose() }; +} + +const notice = { boardId: BOARD, taskId: "t1", attempt: 1, ending: "completed" as const, output: "fact" }; + +describe("a notice its session turns away (defer)", () => { + it("runs nothing, stays owed, and records when it was turned away", async () => { + const h = host(() => true); + try { + await h.act("seed", { id: "t1" }); + const before = Date.now(); + await h.act("onTaskSettled", notice, true); + expect(h.heard).toEqual([]); + const metadata = (await h.act("read", { id: "t1" })) as Record; + expect(metadata["noticeOwed:1:completed"]).toMatchObject({ attempt: 1, ending: "completed" }); + expect(metadata["noticeDeferred:1:completed"]!.at).toBeGreaterThanOrEqual(before); + } finally { + await h.dispose(); + } + }); + + it("runs and is acknowledged when it isn't turned away", async () => { + const h = host(() => false); + try { + await h.act("seed", { id: "t1" }); + await h.act("onTaskSettled", notice, true); + expect(h.heard).toEqual([expect.objectContaining({ taskId: "t1", ending: "completed" })]); + const metadata = (await h.act("read", { id: "t1" })) as Record; + expect(metadata["noticeOwed:1:completed"] ?? null).toBeNull(); + } finally { + await h.dispose(); + } + }); +}); diff --git a/packages/orchestration/test/tasks/pass.test.ts b/packages/orchestration/test/tasks/pass.test.ts index 0c856b7d89..85844c3a50 100644 --- a/packages/orchestration/test/tasks/pass.test.ts +++ b/packages/orchestration/test/tasks/pass.test.ts @@ -78,6 +78,29 @@ describe("a board's pass", () => { for (let i = 1; i < gaps.length; i += 1) expect(gaps[i]!).toBeGreaterThan(gaps[i - 1]!); }); + it("doesn't end a notice a busy session turned away lately, however old its due time (FIX-2113)", async () => { + // Due a day and a minute ago, past the 24 h maximum debt age; turned away a minute ago. + const day = 86_400_000; + const clock = { t: day + 120_000 }; + const turnedAway = ended(0); + const marker = (turnedAway.metadata as Record>)["noticeOwed:1:completed"]!; + const ref = board( + [{ ...turnedAway, metadata: { "noticeOwed:1:completed": marker, "noticeDeferred:1:completed": { at: clock.t - 60_000 } } } as Task], + clock + ); + const sent: unknown[] = []; + await payBoardDebts(session(sent) as never, ref as never); + // Still owed and sent again, never settled undeliverable. + const metadata = ref.get("t1")!.metadata as Record; + expect(metadata["noticeUndeliverable:1:completed"]).toBeUndefined(); + expect(metadata["noticeOwed:1:completed"]).toMatchObject({ attempt: 1, ending: "completed" }); + expect(sent).toHaveLength(1); + // The same notice never turned away is past the age, and ends. + const old = board([ended(0)], clock); + await payBoardDebts(session([]) as never, old as never); + expect((old.get("t1")!.metadata as Record)["noticeUndeliverable:1:completed"]).toBeDefined(); + }); + it("says when its next resend falls due, so the wake waits for it", async () => { const clock = { t: 10_000 }; const ref = board([ended(0)], clock); diff --git a/packages/orchestration/test/tasks/task-notice.test.ts b/packages/orchestration/test/tasks/task-notice.test.ts index 59109cba26..3ac4ac6d01 100644 --- a/packages/orchestration/test/tasks/task-notice.test.ts +++ b/packages/orchestration/test/tasks/task-notice.test.ts @@ -17,14 +17,18 @@ import { cancelledDependents, clearNotice, decideNotice, + deferredNoticePatch, deliveredByWait, NOTICE_DUE_MS, noticeDueAt, isNoticeOwed, noticeKey, + noticeTimes, noticeText, owedNotices, recordEnding, + resendNowPatch, + resentRecord, withoutNoticeMarkers, type TaskNotice } from "../../src/tasks/notice/task-notice"; @@ -108,6 +112,32 @@ describe("recordEnding: the row to write, with the notice owed", () => { expect(noticeDueAt(row({ metadata: { "noticeOwed:1:completed": { attempt: 1, ending: "completed" } } }))).toBe(0); }); + it("drops an owed notice's resend backoff, so the next pass sends it at once (FIX-2113)", () => { + const written = recordEnding(row({ updatedAt: 10_000 }), { kind: "completed", output: 1 }); + const notice = { attempt: 1, ending: "completed" } as const; + // Sent again three times: the next send backs off well past its due time. + const backedOff = { ...written, metadata: { ...written.metadata, ...resentRecord({ ...written, metadata: { ...written.metadata, "noticeResent:1:completed": { at: 20_000, n: 2 } } }, notice, 30_000) } }; + expect(noticeDueAt(backedOff)).toBeGreaterThan(30_000); + const patch = resendNowPatch(backedOff); + expect(patch).toEqual({ "noticeResent:1:completed": null }); + const now = { ...backedOff, metadata: { ...backedOff.metadata, ...patch } }; + expect(noticeDueAt(now)).toBe(10_000 + NOTICE_DUE_MS); + // The owed marker itself stays; a row with no resend record needs nothing. + expect(isNoticeOwed(now, notice)).toBe(true); + expect(resendNowPatch(written)).toEqual({}); + }); + + it("counts a turned-away notice's age from when it was turned away, not from its due time (FIX-2113)", () => { + const written = recordEnding(row({ updatedAt: 10_000 }), { kind: "completed", output: 1 }); + const notice = { attempt: 1, ending: "completed" } as const; + expect(noticeTimes(written, notice)?.since).toBe(10_000 + NOTICE_DUE_MS); + const turnedAway = { ...written, metadata: { ...written.metadata, ...deferredNoticePatch(written, notice, 90_000) } }; + expect(noticeTimes(turnedAway, notice)).toMatchObject({ due: 10_000 + NOTICE_DUE_MS, since: 90_000 }); + // Still owed, and a row that owes nothing is left as it is. + expect(isNoticeOwed(turnedAway, notice)).toBe(true); + expect(deferredNoticePatch(row(), notice, 90_000)).toEqual({}); + }); + it("marks a park with its question (BR-25), and keeps it when the task completes later", () => { const parked = recordEnding(row({ status: "parked" }), { kind: "parked", question: "Which region?", quiet: false }); const completed = recordEnding({ ...parked, status: "completed", attempts: 2, output: "eu" }, { diff --git a/packages/workforce/README.md b/packages/workforce/README.md index b7bb21f94a..e2be82e1d9 100644 --- a/packages/workforce/README.md +++ b/packages/workforce/README.md @@ -2202,7 +2202,7 @@ tool a sent message is a call to), `WORKER_MESSAGE_ENTRY`, `WORKER_MESSAGES_PATT | `createWorkerLookup({ installation })` | Which standard worker a task's name means. Returns `{ find, flowKind, state, filingCheck }`. `find(name)` answers `{ found: true, flowId }`, or `{ found: false, reason, message }` with `reason` `not-found` or `takes-no-tasks` (a worker whose flow declares no `work` task entry). `flowKind` and `state` are a `per-task` dispatcher's targets: the flow of the worker the task's `assignee` names, and a starting state naming that worker. | | `WORKER_TASK_ENTRY` | `"work"`, the task entry a worker flow takes tasks through. | | `workerTaskEntry({ name, turn \| block })` / `WorkerTaskEntryOptions` / `WorkerTaskBlockInput` / `conversationLedgers` / `filingConversationBoard` | The `work` task entry for a worker flow of your own: `task: { actions: { work: workerTaskEntry({ name, turn }) } }`. Each task a coordinator files for one of its workers runs as one `turn` (`{ message }` in, the answer out), or, with `block` instead (one or the other, refused at build otherwise), as a block handed the task itself plus `claim` (`{ collectionId, partition, incarnationId }`, the board the task was claimed from, never read from the task), in a task session of its own, and the conversation that filed it is told when it ends. A flow whose `work` entry is built any other way takes no conversation task: `listDelegates` reports it as taking `nothing`, and a filing for it is refused. `conversationLedgers(ledgerId, ctx, partition)` finds the board of the conversation a task came from; pass it to a harness manager that takes tasks from conversations. `filingConversationBoard(ctx)` answers, in a task session, that conversation's board with the entry and flow that run it there (`{ tasks, runEntry, filingFlow, filingSession }`), for a door over the session's run (`harnessSessionDoor`'s `board`). A coding run's `projectWorkspace()` finds its project from the filing workstream, then the project of the conversation that filed it, then the task's `input.project` (`{ visibility, id }`, checked against the run's own user), then the `board` claim. | -| `STEER_TURN_ENTRY` / `steerTurnEntry({ name, turn \| block, concurrency? })` / `SteerTurnEntryOptions` / `startOwedSteerTurn(ctx, row, flowKind?)` | The turn a task is owed for a message no request of its attempt took: one that landed after its worker's last model call, or while it waited on pieces it filed. Declare it beside `work` on a worker flow of your own, with the same `turn` or `block`: `internal: { actions: { [STEER_TURN_ENTRY]: steerTurnEntry({ name, turn }) } }`. A `turn` gets the messages as its message; a `block` gets the task as the board packs it, messages included. Its answer is the task's result, as a woken turn's is. `startOwedSteerTurn` starts that turn in the session the row's run link names. The turn runs even if that first start is lost. A coding flow's `harnessSessionDoor` takes it as `onSteered`. | +| `STEER_TURN_ENTRY` / `steerTurnEntry({ name, turn \| block, concurrency? })` / `SteerTurnEntryOptions` / `startOwedSteerTurn(ctx, row, flowKind?)` | The turn a task is owed for a message no request of its attempt took: one that landed after its worker's last model call, or while it waited on pieces it filed. Declare it beside `work` on a worker flow of your own, with the same `turn` or `block`: `internal: { actions: { [STEER_TURN_ENTRY]: steerTurnEntry({ name, turn }) } }`. A `turn` gets the messages as its message; a `block` gets the task as the board packs it, messages included. The messages it takes also show in the session as lines from their senders, so a later turn there reads them. Its answer is the task's result, as a woken turn's is. `startOwedSteerTurn` starts that turn in the session the row's run link names. The turn runs even if that first start is lost. A coding flow's `harnessSessionDoor` takes it as `onSteered`. | | `defineWorkerInventoryCollection()` | The worker inventory: one org-scoped row per registered worker, at `inventory/workers/`. Takes no options; install what it returns under a block's `resources`. | | `WorkerInventoryRow` | One row of the inventory. | | `workerInventoryRowSchema` | The Zod schema behind the row type. Closed: an undeclared key is dropped on the way in. | diff --git a/packages/workforce/src/conversation-board/board.ts b/packages/workforce/src/conversation-board/board.ts index 874e4cae26..3b7b41e1cb 100644 --- a/packages/workforce/src/conversation-board/board.ts +++ b/packages/workforce/src/conversation-board/board.ts @@ -89,7 +89,7 @@ import { type AssigneeRoster, type AssigneeRosterSource } from "@flow-state-dev/orchestration"; -import { currentWorkerClaim, taskBoard } from "@flow-state-dev/orchestration/task-board"; +import { afterTaskCheck, currentWorkerClaim, taskBoard } from "@flow-state-dev/orchestration/task-board"; import { TaskCapExceededError, TaskFilingRefusedError, @@ -106,6 +106,7 @@ import { NOTICE_DUE_MS, noticeDueAt, filerResolver, + resendNowPatch, type Task, type TaskCollectionRef, type TaskDispatcher, @@ -775,7 +776,32 @@ export function defineConversationBoard(options: ConversationBoardOptions) { ); /** The flow's planning: its turns, wrapped so the steps they file run in their request (FIX-2076). */ - const planning = definePlanning({ newSession: stepToOwnSession }); + /** + * After a request worked its plan and answered (FIX-2113): the notices this + * session turned away while that turn waited on its own step + * (`anotherTurnWaitsOnItsPlan`) are sent now, past their backoff, by the + * session's pass. After the task's check, so they run after this turn, + * one turn at a time. + */ + const sendWaitingNotices = handler({ + name: "conversation-plan-send-waiting-notices", + inputSchema: z.unknown(), + resources: { ...conversationLedgerResources }, + execute: async (_input, ctx) => { + await afterTaskCheck(ctx as never, async () => { + const own = await conversationLedgerAt(ctx as never, await filingSessionIdOf(ctx.session)); + const owed = (own?.list() ?? []).filter((row) => noticeDueAt(row) !== undefined); + if (own === undefined || owed.length === 0) return; + for (const row of owed) { + const patch = resendNowPatch(row); + if (Object.keys(patch).length > 0) await own.patchMetadata(row.id, patch); + } + await triggerPass(ctx as never); + }); + } + }); + + const planning = definePlanning({ newSession: stepToOwnSession, afterPlan: sendWaitingNotices }); /** The worker this request's turn was verified to run as, when steps are worked on this flow. */ const stepper = (ctx: BlockContext): string | undefined => (planning.wired() ? verifiedWorkerOf(ctx.session) : undefined); diff --git a/packages/workforce/src/conversation-board/plan.ts b/packages/workforce/src/conversation-board/plan.ts index 599c81294a..4e9d7a7d02 100644 --- a/packages/workforce/src/conversation-board/plan.ts +++ b/packages/workforce/src/conversation-board/plan.ts @@ -126,6 +126,71 @@ export function isAnsweringTurn(ctx: BlockContext): boolean { return (ctx.request?.state as Record | undefined)?.[PLAN_ANSWERING_STATE] === true; } +/** + * Session state: the requests whose plan has a step in a session of its own + * to wait on (FIX-2113), each from its turn's end until it answers, keyed by + * request id. Each request writes and clears only its own key, so two that + * plan in one session never overwrite each other. A request that dies leaves + * its key, which then holds nothing (`anotherTurnWaitsOnItsPlan` asks whether + * it ended), and the next plan in the session clears it. + */ +export const PLAN_WAITING_STATE = "planWaitingRequests"; + +/** The session state a plan writes. */ +const planWaitingSessionState = z.object({ [PLAN_WAITING_STATE]: z.record(z.string(), z.literal(true)).optional() }); + +/** The requests the session's state marks as waiting on a step of their own plan. */ +function markedRequests(ctx: BlockContext): string[] { + const marked = (ctx.session?.state as Record | undefined)?.[PLAN_WAITING_STATE]; + return marked !== null && typeof marked === "object" ? Object.keys(marked) : []; +} + +/** + * Whether another request in this session is working a plan with a step in a + * session of its own, and hasn't answered (FIX-2113). That request's turn is + * not over while it waits on the step, and one turn of a task runs at a time + * in its session, so a notice's turn or a steer's turn waits until it ends. + * Read off the session's state, never the plan ledger: a request reads a + * ledger as the snapshot it first took, and its own plan reads it later. + */ +export async function anotherTurnWaitsOnItsPlan(ctx: BlockContext): Promise { + const others = markedRequests(ctx).filter((requestId) => requestId !== ctx.request?.identity?.id); + if (others.length === 0) return false; + // A host with no record of ended requests holds no durable wait, and a step + // filed `session: "new"` is refused there, so no request marks the session; + // reading a mark anyway, take it as live. + const ended = ctx.requestHost?.waits?.ended; + if (ended === undefined) return true; + // A request that died holds nothing. + const answers = await ended(others); + return others.some((requestId) => answers[requestId] !== true); +} + +/** + * Mark the session as waiting on the running request's plan (FIX-2113), + * under its own key; first, drop the keys of requests that have ended. A key + * of a request that is interrupted and not yet driven again reads as ended + * and is dropped too: if the sweeper drives it into its step's wait later, it + * waits unmarked. That needs a crash between this mark and the drain's park, + * with a second plan in the window between, and is accepted. + */ +export async function markPlanWaiting(ctx: BlockContext): Promise { + const own = ctx.request.identity.id; + const others = markedRequests(ctx).filter((requestId) => requestId !== own); + const ended = ctx.requestHost?.waits?.ended; + if (ended !== undefined && others.length > 0) { + const answers = await ended(others); + for (const requestId of others) if (answers[requestId] === true) await ctx.session.deleteStateRecord(PLAN_WAITING_STATE, requestId); + } + await ctx.session.setStateRecord(PLAN_WAITING_STATE, own, true); +} + +/** Clear the running request's own mark (FIX-2113): it answered. Another request's is left. */ +export async function clearPlanWaiting(ctx: BlockContext): Promise { + const own = ctx.request.identity.id; + if (markedRequests(ctx).includes(own)) await ctx.session.deleteStateRecord(PLAN_WAITING_STATE, own); +} + /** Whether the running request is in a step's turn. */ export function isSteppingTurn(ctx: BlockContext): boolean { return (ctx.request?.state as Record | undefined)?.[PLAN_STEPPING_STATE] === true; @@ -194,6 +259,12 @@ export interface PlanningOptions { * every step runs inline. */ readonly newSession?: BlockDefinition; + /** + * Run as the last step of a request that worked its plan, once its turn + * answered: a `.tap()`. The conversation board's sends the notices that + * waited for this turn (FIX-2113). + */ + readonly afterPlan?: BlockDefinition; } /** Build one flow's planning. */ @@ -267,9 +338,23 @@ export function definePlanning(options: PlanningOptions = {}): Planning { name: "plan-decide", inputSchema: z.unknown(), resources: { ...planLedgerResources }, + sessionStateSchema: planWaitingSessionState, execute: async (_input, ctx) => { const own = await ownPlanLedger(ctx as never); - await ctx.request.patchState({ [PLAN_OWED_STATE]: (own?.list() ?? []).some(isWorkable) } as never); + const workable = (own?.list() ?? []).filter(isWorkable); + await ctx.request.patchState({ [PLAN_OWED_STATE]: workable.length > 0 } as never); + // A step in a session of its own: this request waits on it, and its turn isn't over until it answers. + if (workable.some((row) => row.session === "new")) await markPlanWaiting(ctx as never); + } + }); + + /** The request answered after its plan: its turn is over (FIX-2113). */ + const endPlanWaiting = handler({ + name: "plan-end-waiting", + inputSchema: z.unknown(), + sessionStateSchema: planWaitingSessionState, + execute: async (_input, ctx) => { + await clearPlanWaiting(ctx as never); } }); @@ -304,12 +389,14 @@ export function definePlanning(options: PlanningOptions = {}): Planning { // In order: each step sees the one before it end. concurrency: 1 }); - const answer = sequencer({ name: `${turn.name}-plan-answer`, inputSchema: z.unknown() }) + let answer = sequencer({ name: `${turn.name}-plan-answer`, inputSchema: z.unknown() }) .tap(notePlanStart) .tap(board.drain) .step(briefAnswer) .step(turn) - .tap(endAnswer); + .tap(endAnswer) + .tap(endPlanWaiting); + if (options.afterPlan !== undefined) answer = answer.tap(options.afterPlan); const block = sequencer({ name: `${turn.name}-planned`, inputSchema: z.object({ message: z.string() }) }) .step(turn) .tap(decidePlan) diff --git a/packages/workforce/src/conversation-board/session-board.ts b/packages/workforce/src/conversation-board/session-board.ts index a7b56f7f30..b31f5d9f40 100644 --- a/packages/workforce/src/conversation-board/session-board.ts +++ b/packages/workforce/src/conversation-board/session-board.ts @@ -43,6 +43,7 @@ import type { BlockContext, BlockDefinition } from "@flow-state-dev/core/types"; import type { NoticePolicy } from "@flow-state-dev/orchestration/tasks"; import { z } from "zod"; +import { PLAN_WAITING_STATE } from "./plan"; import { DELEGATE_SERVER_OWNED, delegateStateShape, type DelegateRecord } from "../delegates/delegate-list"; import { defineWorkerDelegates, soleTaskDelegate, type TaskDelegates } from "../delegates/worker-delegates"; import { messageHopStateShape } from "../messages/hop"; @@ -97,7 +98,10 @@ export const SESSION_BOARD_SERVER_OWNED: readonly string[] = [ ...CONVERSATION_BOARD_SERVER_OWNED, ...WORKER_MESSAGE_SERVER_OWNED, // The `work` entry writes it when it starts a routed task in a session the routing conversation doesn't own (FIX-2083). - ROUTED_FROM_STATE_KEY + ROUTED_FROM_STATE_KEY, + // A plan writes it while it waits on a step in a session of its own (FIX-2113). A seeded key + // naming a request that never ends here would hold the session's notices with no live wait. + PLAN_WAITING_STATE ]; /** What a session board is built from. */ diff --git a/packages/workforce/src/conversation-board/steer.ts b/packages/workforce/src/conversation-board/steer.ts index eb7ef280d5..7a4fe6f06d 100644 --- a/packages/workforce/src/conversation-board/steer.ts +++ b/packages/workforce/src/conversation-board/steer.ts @@ -134,7 +134,9 @@ export type SteerTurnEntryOptions = { * `turn` or `block`. * * A `turn` is handed the steers as its message; a `block` is handed the task - * as the board packs it, with the claim. A start that finds nothing to do + * as the board packs it, with the claim. The steers it takes are shown in + * the session as lines from their senders, so a later turn there reads them + * too (FIX-2113). A start that finds nothing to do * runs nothing. The wake is cleared once the task no longer waits on a steer. * `concurrency` defaults to the session's notices, so this runs once the * request working the task has ended. @@ -150,7 +152,10 @@ export function steerTurnEntry(options: SteerTurnEntryOptions & { readonly concu // Admitted, the claim is stamped and names the row; otherwise there is nothing to run. const running = await runningTaskOf(sessionLedger, ctx as never); if (running === undefined) return SKIP; - const steers = await takeRunningSteers(running, `turn:${ctx.request.identity.id}`); + // Shown in the session as it takes them, as a model call's take shows + // them (FIX-2113): the turn's own message isn't kept, so a later turn + // here (the notice of the work the task filed) reads them only there. + const steers = await takeRunningSteers(running, `turn:${ctx.request.identity.id}`, ctx); if (steers.length === 0) return SKIP; return { steers, taskId: running.row.id }; } @@ -204,7 +209,12 @@ export function steerTurnEntry(options: SteerTurnEntryOptions & { readonly concu return { inputSchema: z.object({}).passthrough(), block: sequencer({ name, inputSchema: z.unknown() }) - .tap(admitOwnTask(`${name}-admission`, true)) + // One turn of a task at a time in its session (FIX-2113): while another + // request here waits on a step of its own plan, the admission admits + // nothing, so the take finds no task and this runs nothing, leaving the + // wake. That request's end checks the task, and owes this turn again for + // a steer it didn't take. + .tap(admitOwnTask(`${name}-admission`, true, { unlessAnotherTurnWaits: true })) .step(take as BlockDefinition) .stepIf((taken: unknown) => !skipped(taken), run as BlockDefinition) .tapIf((answer: unknown) => !skipped(answer), noteTaskAnswer) diff --git a/packages/workforce/src/conversation-board/task-admission.ts b/packages/workforce/src/conversation-board/task-admission.ts index 2792465337..68280815d8 100644 --- a/packages/workforce/src/conversation-board/task-admission.ts +++ b/packages/workforce/src/conversation-board/task-admission.ts @@ -25,6 +25,7 @@ import { filedWorkHolds, holdsItsFiler, type Task, type TaskCollectionRef } from import { boardAboveOf } from "./board-above"; import { CHECK_TASK_ENTRY, PIECE_OF, STEER_TURN_ENTRY } from "./board-entries"; import { conversationLedgerAt, ownConversationLedger } from "./ledger"; +import { anotherTurnWaitsOnItsPlan } from "./plan"; /** The running row this session works, on the board above it (`./board-above`), or none. */ async function ownTaskRow(ctx: BlockContext): Promise<{ ref: TaskCollectionRef; row: Task } | undefined> { @@ -103,12 +104,16 @@ export function checkTaskEntry(concurrency: ConcurrencyConfig) { * the conversation ledger (the session kit's resources). * * @param answerCounts Whether the entry's turn answer can be the task's result (a woken turn's can; the check entry, which runs no turn, can't). + * @param options.unlessAnotherTurnWaits Admit nothing while another request in the session waits on a step of its own plan (a steer's turn, FIX-2113). */ -export function admitOwnTask(name: string, answerCounts: boolean) { +export function admitOwnTask(name: string, answerCounts: boolean, options: { readonly unlessAnotherTurnWaits?: boolean } = {}) { return taskRequestAdmission({ name, answerCounts, - find: ownTaskRow, + find: async (ctx) => + // With `unlessAnotherTurnWaits`, nothing while another request here + // waits on a step of its own plan (FIX-2113): its turn isn't over. + options.unlessAnotherTurnWaits === true && (await anotherTurnWaitsOnItsPlan(ctx)) ? undefined : ownTaskRow(ctx), endRules: conversationEndRules }); } diff --git a/packages/workforce/src/conversation-board/task-settled.ts b/packages/workforce/src/conversation-board/task-settled.ts index bd5ace796b..aa4d82929d 100644 --- a/packages/workforce/src/conversation-board/task-settled.ts +++ b/packages/workforce/src/conversation-board/task-settled.ts @@ -57,6 +57,7 @@ import { CONVERSATION_LEDGER_ID, ownConversationLedger } from "./ledger"; import { sendTaskReport } from "./report-owed"; import { noteTaskAnswer } from "@flow-state-dev/orchestration/task-board"; import { admitOwnTask } from "./task-admission"; +import { anotherTurnWaitsOnItsPlan } from "./plan"; /** A notice, as the entries take it. */ export const taskNoticeSchema = z @@ -355,5 +356,12 @@ export function taskSettledEntry(options: TaskSettledOptions) { .tapIf((settled: Settled) => settled.act === "routed", acknowledge); } - return orchestrationTaskSettledEntry({ block: handling, concurrency: AFTER_REPLY_CONCURRENCY }); + return orchestrationTaskSettledEntry({ + block: handling, + concurrency: AFTER_REPLY_CONCURRENCY, + // One turn of a task at a time in its session (FIX-2113): while another + // request here waits on a step of its own plan, the notice stays owed, + // and that request sends it once it answers. + defer: (ctx) => anotherTurnWaitsOnItsPlan(ctx) + }); } diff --git a/packages/workforce/test/steer-while-filed-work-open.test.ts b/packages/workforce/test/steer-while-filed-work-open.test.ts new file mode 100644 index 0000000000..44705346fa --- /dev/null +++ b/packages/workforce/test/steer-while-filed-work-open.test.ts @@ -0,0 +1,267 @@ +/** + * A steer to a task that waits on the work it filed (FIX-2113). The steer's + * own turn runs at once, and the turn that answers after the work ends (the + * notice's) is a later request in the same session: it reads the steer only + * from the session's history, so the steer turn's take must leave it there. + * + * The host is the conversation board's (`./conversation-board-harness`): + * `otto` on the built-in `agent` flow, with `eng.tasker` as its delegate, and + * a scripted model. What each model call was sent is read off the mock. + */ +import { describe, expect, it } from "vitest"; +import type { Task } from "@flow-state-dev/orchestration/tasks"; +import { mockGenerator } from "@flow-state-dev/testing"; +import { boardWorkers, bootBoardHost, messageOf } from "./conversation-board-harness"; +import type { WorkerManifest } from "../src/manifest"; +import { anotherTurnWaitsOnItsPlan, clearPlanWaiting, markPlanWaiting } from "../src/conversation-board/plan"; + +const workers = (): WorkerManifest[] => [ + ...boardWorkers({ otto: { delegates: ["eng.tasker"] } }), + { id: "chief", declared: { flow: "coordinator", delegates: ["otto"] }, body: "Hand the work to otto.", skills: [] } +]; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Everything one model call was sent, as one string. */ +const sentText = (input: unknown) => JSON.stringify(input).replaceAll("\\n", "\n").replaceAll('\\"', '"'); + +async function until(check: () => Promise, what: string, ms = 5_000) { + for (const deadline = Date.now() + ms; Date.now() < deadline; await sleep(10)) if (await check()) return; + throw new Error(`timed out waiting for ${what}`); +} + +describe("a steer while the task waits on the work it filed", () => { + it("reaches the turn that answers after the work ends, and is taken once", async () => { + const STEER = "Also use the word ember."; + let release!: () => void; + const holds = new Map([["research", new Promise((resolve) => (release = resolve))]]); + const agentAnswer = mockGenerator({ + script: [ + // The task's turn (one model call, two steps) files the research and ends: the task waits on it. + { toolCalls: [{ toolCallId: "c1", toolName: "createTask", args: { goal: "[hold:research] Find the fact", assignee: "eng.tasker" } }] }, + { text: "Filed the research; waiting on it." }, + // The steer's turn. + { text: "Noted: I'll use the word ember." }, + // The research's notice wakes the turn whose answer is the task's result. + { text: "The post, with the fact and the word ember." } + ] + }); + const h = bootBoardHost({ + standard: workers(), + agentAnswer, + holds, + judgment: mockGenerator({ script: [{ when: () => true, then: { text: "Noted." } }] }) + }); + try { + const conv = await h.conversation("alice", "chief"); + const filing = await h.filingOf("alice", conv); + const filed = await h.act("alice", conv, "createTask_tasks", { goal: "Write the post", assignee: "otto" }); + expect(filed.error, messageOf(filed.error)).toBeUndefined(); + const taskId = (filed.output as { taskId: string }).taskId; + const row = async () => (await h.row("alice", filing, taskId)) as Task | undefined; + await until(async () => (await row())?.waitingOn === "filed", "the task to wait on its research"); + + const steered = await h.act("alice", conv, "updateTask_tasks", { id: taskId, message: STEER }); + expect(steered.output).toEqual({ ok: true }); + await until(async () => agentAnswer.calls.length === 2, "the steer's turn"); + release(); + await h.settled(); + + const done = (await row())!; + expect(done).toMatchObject({ status: "completed", attempts: 1, output: "The post, with the fact and the word ember." }); + // Taken once, by the steer's turn, whose message carried it. + expect(done.steers).toEqual([expect.objectContaining({ message: STEER, takenBy: 1, takenIn: expect.stringMatching(/^turn:/) })]); + expect(sentText(agentAnswer.calls[1]!.input)).toContain(STEER); + // The notice's turn, a later request in the session, was sent the steer + // too: before it, the steer turn's take left it nowhere a later turn reads. + const last = sentText(agentAnswer.calls[2]!.input); + expect(last).toContain("completed by eng.tasker"); + expect(last).toContain(STEER); + } finally { + await h.dispose(); + } + }, 20_000); + + it("runs one turn of the task at a time: the research's turn waits while the steer's turn waits on its own step", async () => { + let release!: () => void; + const holds = new Map([["research", new Promise((resolve) => (release = resolve))]]); + /** The last user message one model call was sent: what that turn is answering. */ + const lastUser = (input: unknown) => { + const s = sentText(input); + return s.slice(s.lastIndexOf('"role":"user"')); + }; + const once = (flag: { done: boolean }, test: (i: unknown) => boolean) => (i: unknown) => { + if (flag.done || !test(i)) return false; + flag.done = true; + return true; + }; + const filedResearch = { done: false }; + const steerStep = { done: false }; + const noticeStep = { done: false }; + const ANSWERING = "The steps you filed for yourself have ended"; + const agentAnswer = mockGenerator({ + script: [ + { when: (i) => lastUser(i).includes(ANSWERING), then: { text: "The post, after the check." } }, + { when: (i) => lastUser(i).includes("You are working a step you filed for yourself"), then: { text: "checked" } }, + { when: (i) => lastUser(i).includes("[step]"), then: { text: "checked in its own session" } }, + // The steer's turn files a step in a session of its own: its request waits on it. + { + when: once(steerStep, (i) => lastUser(i).includes("You were sent the messages below")), + then: { toolCalls: [{ toolCallId: "s1", toolName: "createTask", args: { goal: "[step] Check the draft", assignee: "otto", session: "new" } }] } + }, + { when: (i) => lastUser(i).includes("You were sent the messages below"), then: { text: "Waiting on the check." } }, + // The research's turn files one too. + { + when: once(noticeStep, (i) => lastUser(i).includes("completed by eng.tasker")), + then: { toolCalls: [{ toolCallId: "s2", toolName: "createTask", args: { goal: "[step] Check the fact", assignee: "otto", session: "new" } }] } + }, + { when: (i) => lastUser(i).includes("completed by eng.tasker"), then: { text: "Waiting on the fact check." } }, + { + when: once(filedResearch, (i) => lastUser(i).includes("Write the post")), + then: { toolCalls: [{ toolCallId: "c1", toolName: "createTask", args: { goal: "[hold:research] Find the fact", assignee: "eng.tasker" } }] } + }, + { when: (i) => lastUser(i).includes("Write the post"), then: { text: "Waiting on research." } } + ] + }); + // When each model call was sent, and what it answers. The steer's step + // takes 9 s, long enough that the turned-away notice's next resend backs + // off past the moment the steer's turn answers. + const sentAt: Array<{ at: number; asked: string }> = []; + let holdStep = false; + const h = bootBoardHost({ + standard: workers(), + agentAnswer, + holds, + observeTools: (_block, _names, input) => { + const asked = lastUser(input); + sentAt.push({ at: Date.now(), asked }); + if (asked.includes("[step] Check the draft") && !asked.includes(ANSWERING)) holdStep = true; + }, + afterModelCall: async () => { + if (!holdStep) return; + holdStep = false; + await sleep(9_000); + }, + durable: true, + judgment: mockGenerator({ script: [{ when: () => true, then: { text: "Noted." } }] }) + }); + try { + const conv = await h.conversation("alice", "chief"); + const filing = await h.filingOf("alice", conv); + const filed = await h.act("alice", conv, "createTask_tasks", { goal: "Write the post", assignee: "otto" }); + expect(filed.error, messageOf(filed.error)).toBeUndefined(); + const taskId = (filed.output as { taskId: string }).taskId; + const row = async () => (await h.row("alice", filing, taskId)) as Task | undefined; + await until(async () => (await row())?.waitingOn === "filed", "the task to wait on its research"); + expect((await h.act("alice", conv, "updateTask_tasks", { id: taskId, message: "Also use the word ember." })).output).toEqual({ ok: true }); + await until(async () => steerStep.done, "the steer's turn to file its step"); + await until(async () => sentAt.some((call) => call.asked.includes("[step] Check the draft")), "the step to start"); + // A second steer while the steer's turn waits on its step: no turn of its own runs beside it. + expect((await h.act("alice", conv, "updateTask_tasks", { id: taskId, message: "Also add a title." })).output).toEqual({ ok: true }); + await sleep(2_500); + // The research ends while the steer's turn waits on its step. + release(); + // Done well before a lease (120 s) could free a stalled task. + await until(async () => (await row())?.status === "completed", "the task to complete", 20_000); + await h.settled(); + + const asked = agentAnswer.calls.map((c) => lastUser(c.input)); + const at = (words: string, from = 0) => asked.findIndex((text, i) => i >= from && text.includes(words)); + const steerTurn = at("You were sent the messages below"); + const steerAnswer = at(ANSWERING, steerTurn); + const researchTurn = at("completed by eng.tasker"); + // The research's turn ran only once the steer's turn had answered after its step. + expect(steerAnswer, JSON.stringify(asked)).toBeGreaterThan(steerTurn); + expect(researchTurn, JSON.stringify(asked)).toBeGreaterThan(steerAnswer); + // The second steer was read no earlier than the steer's turn's answer: by that answer, or a turn after it. + const secondSteer = agentAnswer.calls.findIndex((call) => sentText(call.input).includes("Also add a title.")); + expect(secondSteer, JSON.stringify(asked)).toBeGreaterThanOrEqual(steerAnswer); + // At once: the steer's turn sends it on as it answers, not at the notice's next resend. + const answered = sentAt.find((call) => call.asked.includes(ANSWERING))!.at; + const heard = sentAt.find((call) => call.asked.includes("completed by eng.tasker"))!.at; + expect(heard - answered).toBeLessThan(2_500); + // And it read the steer, from the session. + expect(sentText(agentAnswer.calls[researchTurn]!.input)).toContain("Also use the word ember."); + // Its own step was worked, and its answer is the task's result. + expect(at(ANSWERING, researchTurn)).toBeGreaterThan(researchTurn); + expect((await row())).toMatchObject({ status: "completed", attempts: 1, output: "The post, after the check." }); + } finally { + await h.dispose(); + } + }, 40_000); +}); + +describe("whether another turn of the session waits on its plan", () => { + /** A context as the check reads it: the session's marks, the running request, and the host's record of ended requests. */ + const ctxWith = (marked: string[], ended?: Record) => + ({ + session: { state: { planWaitingRequests: Object.fromEntries(marked.map((id) => [id, true])) } }, + request: { identity: { id: "req-me" } }, + requestHost: + ended === undefined + ? {} + : { waits: { ended: async (ids: readonly string[]) => Object.fromEntries(ids.map((id) => [id, ended[id] === true])) } } + }) as never; + + it("is true for another request's live mark, and false for its own, none, or one whose request ended", async () => { + expect(await anotherTurnWaitsOnItsPlan(ctxWith([], {}))).toBe(false); + expect(await anotherTurnWaitsOnItsPlan(ctxWith(["req-me"], {}))).toBe(false); + expect(await anotherTurnWaitsOnItsPlan(ctxWith(["req-other"], { "req-other": false }))).toBe(true); + // The request that marked it died: it holds nothing. + expect(await anotherTurnWaitsOnItsPlan(ctxWith(["req-other"], { "req-other": true }))).toBe(false); + // Two marks: one dead, one live. + expect(await anotherTurnWaitsOnItsPlan(ctxWith(["req-dead", "req-live"], { "req-dead": true }))).toBe(true); + }); + + it("keeps each request's mark its own: one's end leaves another's, and the gate stays shut until both end", async () => { + // One session's state, written as the engine's ops write it. + const state: Record = {}; + const done = new Set(); + const session = { + state, + async setStateRecord(field: string, key: string, value: unknown) { + state[field] = { ...((state[field] as object | undefined) ?? {}), [key]: value }; + return true; + }, + async deleteStateRecord(field: string, key: string) { + const { [key]: _gone, ...rest } = (state[field] as Record | undefined) ?? {}; + state[field] = rest; + return true; + }, + async patchState(updates: Record) { + Object.assign(state, updates); + return true; + } + }; + const requestHost = { waits: { ended: async (ids: readonly string[]) => Object.fromEntries(ids.map((id) => [id, done.has(id)])) } }; + const as = (id: string) => ({ session, request: { identity: { id } }, requestHost }) as never; + + await markPlanWaiting(as("req-a")); + await markPlanWaiting(as("req-b")); + expect(await anotherTurnWaitsOnItsPlan(as("req-notice"))).toBe(true); + // A answers first: B's mark stays, and the gate stays shut. + await clearPlanWaiting(as("req-a")); + done.add("req-a"); + expect(Object.keys(state.planWaitingRequests as object)).toEqual(["req-b"]); + expect(await anotherTurnWaitsOnItsPlan(as("req-notice"))).toBe(true); + await clearPlanWaiting(as("req-b")); + done.add("req-b"); + expect(await anotherTurnWaitsOnItsPlan(as("req-notice"))).toBe(false); + + // The later writer answers first: the earlier one's mark stays, and the gate stays shut. + await markPlanWaiting(as("req-e")); + await markPlanWaiting(as("req-f")); + await clearPlanWaiting(as("req-f")); + done.add("req-f"); + expect(Object.keys(state.planWaitingRequests as object)).toEqual(["req-e"]); + expect(await anotherTurnWaitsOnItsPlan(as("req-notice"))).toBe(true); + await clearPlanWaiting(as("req-e")); + done.add("req-e"); + + // A request that died left its key; the next plan drops it and keeps its own. + await markPlanWaiting(as("req-c")); + done.add("req-c"); + await markPlanWaiting(as("req-d")); + expect(Object.keys(state.planWaitingRequests as object)).toEqual(["req-d"]); + }); +});