From 08ad4a64d2542e7cbbe7c99d5887e0ef64029a7d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 00:28:11 +0000 Subject: [PATCH 01/14] feat(kitchen-sink,goals): support.help and the fresh-host and routed-post desks are coordinators (FIX-1792 P1) Kitchen-sink's help desk is a WORKER.md on the coordinator flow (routing: best-fit, fallback: support.general), converted key by key from its MAILBOX.md, which is deleted. The escalation feature is removed, not replaced: the escalate tool and its test, the no-filing control, the escalations board and its panel, and the specialists' instruction to file. The page lists the person's conversation with support.help under Coordinators, draws its lines under their writers' names, and names each delegate still working. The mailbox wiring, notify block and boot guards over stored mailbox sessions go; the goal controls are re-expressed on the coordinator (no-route, no-landing, no-delivery, answers-go-on). The workforce-mailboxes fresh-host and routed-post goals convert onto the coordinator flow (BR-8, BR-9). The routed-post goal retires its followup and context legs: the coordinator's best fit routes a post on its own words and hands a delegate the post alone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- apps/kitchen-sink/README.md | 97 +++-- apps/kitchen-sink/app/page.tsx | 77 ++-- .../components/picked-session-panel.tsx | 118 +++--- apps/kitchen-sink/components/team-panel.tsx | 56 +-- apps/kitchen-sink/e2e/talk-from-page.spec.ts | 261 ++++++-------- apps/kitchen-sink/e2e/workforce-shell.spec.ts | 97 +++-- apps/kitchen-sink/fsdev.config.ts | 138 +------ apps/kitchen-sink/lib/coordinator-control.ts | 40 +++ apps/kitchen-sink/lib/e2e-mock-script.ts | 124 ++----- apps/kitchen-sink/lib/escalate-control.ts | 35 -- .../lib/kitchen-sink-principal.ts | 7 +- apps/kitchen-sink/lib/landing-control.ts | 112 ++++++ .../lib/mailbox-landing-control.ts | 146 -------- apps/kitchen-sink/lib/mailbox-post-control.ts | 44 --- .../kitchen-sink/lib/mailbox-route-control.ts | 20 -- apps/kitchen-sink/lib/mailbox-wake-control.ts | 60 ---- apps/kitchen-sink/lib/models.ts | 6 +- apps/kitchen-sink/lib/rail-sessions.ts | 40 ++- apps/kitchen-sink/lib/workforce-shell.ts | 76 ++-- apps/kitchen-sink/test/client-imports.test.ts | 4 +- .../kitchen-sink/test/e2e-mock-script.test.ts | 204 ++++------- apps/kitchen-sink/test/escalate.test.ts | 271 -------------- apps/kitchen-sink/test/goal-control.test.ts | 28 +- apps/kitchen-sink/test/mailbox-reply.test.ts | 111 +++--- apps/kitchen-sink/test/mailbox-talk.test.ts | 74 ++-- apps/kitchen-sink/test/mailbox-wake.test.ts | 329 +++++------------ apps/kitchen-sink/test/mock-flowstate.ts | 8 +- apps/kitchen-sink/test/named-org.test.ts | 198 ++-------- .../test/picked-session-panel.test.tsx | 175 ++++----- apps/kitchen-sink/test/shell-imports.test.ts | 10 +- apps/kitchen-sink/test/support-desk.test.ts | 90 +++-- .../kitchen-sink/test/workforce-shell.test.ts | 111 ++---- .../kitchen-sink/workforce/blocks/escalate.ts | 138 ------- apps/kitchen-sink/workforce/hire.ts | 136 +++---- apps/kitchen-sink/workforce/mailbox-notify.ts | 36 -- .../teams/support/mailboxes/help/MAILBOX.md | 11 - .../teams/support/workers/accounts/WORKER.md | 3 +- .../teams/support/workers/devices/WORKER.md | 3 +- .../teams/support/workers/fsd/WORKER.md | 3 +- .../teams/support/workers/general/WORKER.md | 4 +- .../teams/support/workers/help/WORKER.md | 9 + apps/kitchen-sink/workforce/workforce.gen.ts | 7 +- .../teams/desk/mailboxes/front/MAILBOX.md | 6 - .../teams/desk/workers/front/WORKER.md | 8 + .../goal.md | 33 +- .../host.mts | 129 +++---- .../run.mts | 246 +++++++------ .../teams/support/mailboxes/help/MAILBOX.md | 8 - .../teams/support/mailboxes/lounge/MAILBOX.md | 6 - .../teams/support/workers/help/WORKER.md | 9 + .../teams/support/workers/lounge/WORKER.md | 8 + .../a-routed-post-gets-one-answer/goal.md | 43 ++- .../a-routed-post-gets-one-answer/host.mts | 150 +++----- .../a-routed-post-gets-one-answer/run.mts | 339 ++++++++---------- 54 files changed, 1571 insertions(+), 2931 deletions(-) create mode 100644 apps/kitchen-sink/lib/coordinator-control.ts delete mode 100644 apps/kitchen-sink/lib/escalate-control.ts create mode 100644 apps/kitchen-sink/lib/landing-control.ts delete mode 100644 apps/kitchen-sink/lib/mailbox-landing-control.ts delete mode 100644 apps/kitchen-sink/lib/mailbox-post-control.ts delete mode 100644 apps/kitchen-sink/lib/mailbox-route-control.ts delete mode 100644 apps/kitchen-sink/lib/mailbox-wake-control.ts delete mode 100644 apps/kitchen-sink/test/escalate.test.ts delete mode 100644 apps/kitchen-sink/workforce/blocks/escalate.ts delete mode 100644 apps/kitchen-sink/workforce/mailbox-notify.ts delete mode 100644 apps/kitchen-sink/workforce/teams/support/mailboxes/help/MAILBOX.md create mode 100644 apps/kitchen-sink/workforce/teams/support/workers/help/WORKER.md delete mode 100644 goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/fixtures/workforce/teams/desk/mailboxes/front/MAILBOX.md create mode 100644 goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/fixtures/workforce/teams/desk/workers/front/WORKER.md delete mode 100644 goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/mailboxes/help/MAILBOX.md delete mode 100644 goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/mailboxes/lounge/MAILBOX.md create mode 100644 goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/workers/help/WORKER.md create mode 100644 goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/workers/lounge/WORKER.md diff --git a/apps/kitchen-sink/README.md b/apps/kitchen-sink/README.md index 56f1017337..a249bc0a1b 100644 --- a/apps/kitchen-sink/README.md +++ b/apps/kitchen-sink/README.md @@ -1,6 +1,6 @@ # Kitchen Sink -The canonical reference application for `@flow-state-dev`, and a Workforce app you can copy. It hires a team, gives that team a mailbox and a board, and shows all of it in a shell whose navigator, roster and board list are imported from `@flow-state-dev/react`. +The canonical reference application for `@flow-state-dev`, and a Workforce app you can copy. It hires a support team with a coordinator in front, and shows it in a shell whose navigator is imported from `@flow-state-dev/react` and whose roster is read through Workforce's client. Kitchen sink is a reference app, not a minimal example. It hosts every subsystem and is where features get tested end to end. For small, focused, copy-paste-able demos see `examples/`. @@ -35,117 +35,106 @@ The `pnpm fsdev` commands below run from `apps/kitchen-sink`, where `fsdev.confi KITCHEN_SINK_TEST_MODE=1 pnpm --filter @flow-state-dev/kitchen-sink dev ``` -The specialists and the assistant all answer with the same placeholder, `Test mode (no scenario sentinel matched).`, but the mailbox around them works as it does with a key: a post in `support.help` is routed to one specialist, and that specialist's answer lands in the mailbox under its name. The scripted route can't read a question, so it goes by a tag instead. A post containing `[route:support.devices]` goes to `support.devices`, and a post with no tag goes to `support.general`, the mailbox's fallback. The end-to-end tests run in this mode. +The specialists and the assistant all answer with the same placeholder, `Test mode (no scenario sentinel matched).`, but the coordinator in front of them works as it does with a key: a post to `support.help` goes to one specialist, and that specialist's answer lands in your conversation under its name. The scripted pick can't read a question, so it goes by a tag instead. A post containing `[route:support.devices]` goes to `support.devices`, and a post with no tag goes to `support.general`, the coordinator's fallback. The end-to-end tests run in this mode. ## What to read first -- `workforce/`: the support team, four specialists and the mailbox that sends each post to one of them. This is the authoring path, and the shortest route to understanding the app. -- `app/page.tsx`: the shell, with one navigator on the left, the stream in the middle, and boards and the roster on the right. +- `workforce/`: the support team, four specialists and the coordinator that sends each post to one of them. This is the authoring path, and the shortest route to understanding the app. +- `app/page.tsx`: the shell, with one navigator on the left, the stream in the middle, and the roster on the right. - `flows/`: the flows the app serves, including `chat-agent`, the assistant the stream talks to. The chat agent either answers in the turn or files the work to a durable board, where a child session picks it up and the result comes back on a later turn. For coordination that happens inside a single request, see the [patterns documentation](../docs/docs/patterns/overview.md); each pattern page carries its own runnable example. The app uses one pattern, the response auditor, which annotates an answer after it's produced. ## The support team (`workforce/`) -The support team is one mailbox and four specialists, declared in files rather than wired in code. +The support team is one coordinator and four specialists, declared in files rather than wired in code. -Open `support.help` in the rail (expand `mailbox`, under Mailboxes, to find it) and ask a question. The mailbox sends each post to the one specialist whose job fits it: `support.devices` for printers, laptops, phones and wifi, `support.accounts` for sign-in and billing, `support.fsd` for questions about building with flow-state-dev, and `support.general` for anything else. You see which specialist is working on it, then its answer as a line under its name, without reloading. The specialist can show as working for a few seconds after its answer appears, while it finishes its run, filing a case for instance. +Open `support.help` in the rail (under Coordinators) and ask a question. The coordinator sends each post to the one specialist whose job fits it: `support.devices` for printers, laptops, phones and wifi, `support.accounts` for sign-in and billing, `support.fsd` for questions about building with flow-state-dev, and `support.general` for anything else. You see which specialist is working on it, then its answer as a line under its name, without reloading. -The rail lists the mailboxes the files declare. A store kept from an earlier version of the app can still hold other mailboxes' sessions; they stay in the store and don't show in the rail. +The rail shows your own conversation with `support.help`. Every user has one, and nobody else reads it. -Each specialist is a `WORKER.md` under `workforce/teams/support/workers/`. Its `description:` is its job, and it is what the mailbox reads to decide who answers: +Each specialist is a `WORKER.md` under `workforce/teams/support/workers/`. Its `description:` is its job, and it is what the coordinator reads to decide who answers: ```md --- description: Printers, laptops, phones, wifi and anything else with a power button. -tools: [post-to-mailbox, escalate] --- You are the support team's devices specialist. Answer in a sentence or two, and say plainly -when you don't know. When a case needs a person, file it with `escalate` and say you did. +when you don't know. ``` -None of the four names a `flow:`, so all of them run on the built-in agent kind. +None of the four names a `flow:`, so all of them run on the built-in agent flow. ### How a post finds its specialist -Routing is two lines in `workforce/teams/support/mailboxes/help/MAILBOX.md`: +The coordinator is a worker too, in `workforce/teams/support/workers/help/WORKER.md`: ```md -routing: - fallback: support.general +--- +description: Ask the support team anything. +flow: coordinator +delegates: [support.devices, support.accounts, support.fsd, support.general] +routing: best-fit +fallback: support.general +--- ``` -and one in `workforce/hire.ts`, where the mailbox kind is built with `routeByPurpose(seats, { model: ROUTE_MODEL })`. For each post from a person, if their last post is still waiting on a specialist, this one goes there too. Otherwise one evaluator call picks a specialist from the four descriptions, reading the mailbox's recent lines along with the post. (An evaluator is a block that answers a typed question with one model call.) If that call fails, `support.general` takes the post. Only the chosen specialist hears it. +`flow: coordinator` puts it on Workforce's coordinator flow, which `workforce/hire.ts` registers with `defineCoordinatorFlow`, naming the built-in agent flow as the one its delegates answer on and `ROUTE_MODEL` as the model it picks with. For each post from a person, if their last post is still waiting on a specialist, this one goes there too. Otherwise one evaluator call picks a specialist from the four descriptions, reading the post. (An evaluator is a block that answers a typed question with one model call.) If that call fails, `support.general` takes the post. Only the chosen specialist hears it. `ROUTE_MODEL` lives in `lib/models.ts`. It has to be a model that can evaluate, and not every chat model can: see [Evaluation models](../docs/docs/fundamentals/models.md#evaluation-models). The specialists answer on the [`chat` intent](../docs/docs/fundamentals/models.md#intents) under `models.intents` in `fsdev.config.ts`, because no `WORKER.md` names a `model:` of its own. -The specialist answers with the mailbox's last 20 lines in view, so "where can I buy it?" finds its "it" even when the laptop came up with a different specialist. Its own conversation for the mailbox keeps only the posts routed to it and its answers, so something said further back, to someone else, may need saying again. - -Take the `routing:` lines out and every specialist hears every post, which is what a mailbox does by default. - -The support desk needs actions to run in process. With `FSD_BULLMQ_DISPATCH=1` (see [Environment variables](#environment-variables)), a post to `support.help` shows up in the mailbox but no specialist answers it. A specialist you talk to directly answers but says it couldn't file the case. - -### When a case needs a person - -A specialist that decides a case needs a person calls `escalate`, a tool in `workforce/blocks/escalate.ts`. It files one row onto the mailbox's `escalations` board through the mailbox's own `fileTask` action, signed with the specialist's own id, and the specialist says so in its answer. The model chooses only what the row says. The case appears in the team panel's `escalations` list, newest first, with its status, in every open tab as soon as it is filed. The list follows `support.help`, the mailbox the specialist files through. A tab holds one stream connection for the list, and a second if `support.help` is also open in the navigator. - -Only the mailbox's members can file there. Any other seat given the tool is told nothing was filed, and nothing is sent to the mailbox. +The pick reads the post on its own, not the conversation around it. A follow-up such as "it fails right after the password", sent after the specialist has answered, is placed on its own words, so it can reach a different specialist than the one that answered the post it follows. Each specialist keeps its own session for your conversation with `support.help`, holding only the posts it was handed and its answers, so it doesn't see what another specialist said. -Nobody works `escalations` in this app, and the boot says so: +Swap `routing: best-fit` for `routing: everyone` and every specialist hears every post and answers it. -``` -[workforce] mailbox "support.help" holds board "escalations" (ledger -"support.help.escalations"), and no flow hired in this call declares it. … -``` - -Filed cases stay `pending` for a person to read. In an app of your own, that warning is how you find out a board has nobody draining it. +The support desk needs actions to run in process. With `FSD_BULLMQ_DISPATCH=1` (see [Environment variables](#environment-variables)), a post to `support.help` shows up in the conversation but no specialist answers it. ### Talking to one specialist -To talk to a specialist directly, expand `agent` under Seats in the rail, open the specialist, and press the **+** on its row (its tooltip says New conversation). That conversation is separate from the mailbox and keeps both sides across a reload. The specialist sees the earlier turns of that conversation and nothing from any other. A case it files with `escalate` from here lands on `support.help`'s `escalations` board, in the team panel's list with the cases filed from the mailbox. +To talk to a specialist directly, press **Talk** on its row in the roster on the right. That conversation is separate from your conversation with `support.help` and keeps both sides across a reload. The specialist sees the earlier turns of that conversation and nothing from any other. ```bash pnpm fsdev run support.devices run -i '{"message":"My phone stopped charging."}' ``` -Under an open specialist, the rail also lists the conversation it keeps for `support.help`. That row is labelled with its session id, a long string starting `dsx_`, not with the mailbox's name. +The rail lists your conversations with specialists under Workers, and with them each specialist's session for your conversation with `support.help`. That row is labelled with its session id, a long string starting `dsx_`. -### Posting to the mailbox over HTTP +### Posting to the coordinator over HTTP -A mailbox is a named session on the one `mailbox` flow, so the flow in the URL is `mailbox` and the mailbox's id goes where a session id goes: +A conversation with `support.help` is a session on the `coordinator` flow that names it. Open one, then post to it through the flow's `run` action: ```bash -curl -X POST localhost:3000/api/flows/mailbox/support.help/actions/post \ +curl -X POST localhost:3000/api/flows/coordinator/sessions \ -H 'content-type: application/json' \ - -d '{"userId":"devuser","input":{"body":"My phone stopped charging."}}' + -d '{"userId":"devuser","state":{"workerId":"support.help"}}' + +curl -X POST localhost:3000/api/flows/coordinator/actions/run \ + -H 'content-type: application/json' \ + -d '{"userId":"devuser","sessionId":"","input":{"message":"My phone stopped charging."}}' ``` -The call answers `202` with the request it started. The post, and the specialist's answer after it, land on the mailbox's session a moment later. Whatever `userId` you send, this app runs the call as `devuser` (see [One organization](#one-organization)). +The post is kept as your turn, and the specialist's answer lands on the same session a moment later. Whatever `userId` you send, this app runs the call as `devuser` (see [One organization](#one-organization)). -To read the lines back, read the session's items: +To read the lines back, read the session's messages: ```bash -curl 'localhost:3000/api/flows/sessions/support.help/state?include_items=true&item_types=component' +curl 'localhost:3000/api/flows/sessions//state?include_items=true&item_types=message' ``` -Each item whose `component` is `mailbox-post` is one line: its `data` carries the `body`, and an `author` when a specialist wrote it. The `mailbox-route` items beside them record which specialist each post went to. +Your posts are the `user` messages. Each specialist's answer is a message whose `agentName` is the specialist's id. The `coordinator-route` component items beside them record which specialist each post went to. ### Where the pieces come from -Tools, blocks and capabilities come from files too: a file under `workforce/blocks/` or a `resources/` folder becomes an entry in `workforce/workforce.gen.ts` when you run `fsdev gen`. That module is committed, so the code an app can run is fixed when you run the command, which is what lets a bundler see it. The roster is read at boot: `hireKitchenSinkWorkforce()` walks `workforce/teams/` and hires a seat per `WORKER.md`. Adding a specialist means a folder, a line in the mailbox's `members:`, and a restart. Adding a tool means a file and `fsdev gen`. +Tools, blocks and capabilities come from files too: a file under `workforce/blocks/` or a `resources/` folder becomes an entry in `workforce/workforce.gen.ts` when you run `fsdev gen`. That module is committed, so the code an app can run is fixed when you run the command, which is what lets a bundler see it. The roster is read at boot: `buildKitchenSinkWorkforce()` walks `workforce/teams/` and reads a worker per `WORKER.md`. Adding a specialist means a folder, a line in the coordinator's `delegates:`, and a restart. Adding a tool means a file and `fsdev gen`. -Mailboxes are opened at boot, and opening is idempotent. Re-opening is not a migration: an open mailbox keeps the members and charter it was opened with, so on a persistent store a new specialist doesn't join a mailbox that is already open. `boards:` and `routing:` are the exceptions, read from the file on every boot. +A conversation copies the coordinator's `delegates:` the first time it routes a post, and keeps its own list after that, so a specialist added to the file later reaches new conversations, not ones already under way. A specialist's answer does not wake the other specialists. -Which organization the mailbox sessions land in comes from the caller's verified identity, not from anything a file declares. If you add authentication, open them as a caller whose identity already carries the organization you want: [which organization a mailbox runs in](../docs/docs/workforce/mailboxes.md#which-organization-a-mailbox-runs-in). - ### One organization -This app runs as one organization, `kitchen-sink`, and one user, `devuser`. Both are set in `fsdev.config.ts` by a `resolvePrincipal` that reads nothing from the request. It applies to every flow that doesn't bring its own resolver, which means every page, seat and mailbox, so nobody calling the app can pick another organization. It is a stand-in for real sign-in, and it means **anyone who can open a deployed copy of this app can post to its mailbox and talk to its seats, on your model key**. If you deploy it somewhere other people can reach, put sign-in in front of it first. - -**If the app refuses to start with `[workforce] this store was written before kitchen-sink ran as organization "kitchen-sink"`**, the store holds mailbox sessions from another organization, and the message names each one with the organization it belongs to. That data can't be carried over, so delete the store and restart. For the local filesystem profile, delete `.fsdev/data` (or the whole `.fsdev` folder). For Postgres, point `FSD_DB_URL` at an empty database. +This app runs as one organization, `kitchen-sink`, and one user, `devuser`. Both are set in `fsdev.config.ts` by a `resolvePrincipal` that reads nothing from the request. It applies to every flow that doesn't bring its own resolver, which means every page and every worker, the coordinator included, so nobody calling the app can pick another organization. It is a stand-in for real sign-in, and it means **anyone who can open a deployed copy of this app can post to its coordinator and talk to its specialists, on your model key**. If you deploy it somewhere other people can reach, put sign-in in front of it first. ### Hiring while the app runs @@ -166,7 +155,7 @@ The admin hire goes over HTTP because the admin flow checks a token and the CLI The admin action has a credential of its own, but not an organization of its own. Every token has to name `kitchen-sink`. An entry naming any other organization is refused when the app starts, and the refusal is logged, so a token can never quietly administer an organization this app doesn't serve. A seat the admin action hires belongs to the operator who hired it. -The admin flow also has a `fire` action, which removes a seat the admin action hired. A hired seat doesn't show in the rail and doesn't join `support.help`: a mailbox's members are the ones its file names. The page has no control for hiring. +The admin flow also has a `fire` action, which removes a seat the admin action hired. A hired seat doesn't show in the rail and doesn't join `support.help`: a coordinator's delegates are the ones its file names. The page has no control for hiring. The seat answers any configured admin token, not only the one that hired it: every token resolves to the same admin principal, and the seat is pinned to that principal. It belongs to the organization and to the admin user, so its address carries both. The seat carries the admin flow's resolver, so a request with no token, or a token it doesn't recognize, gets `401` from that resolver before the pin is checked. Call it with any configured token: @@ -181,8 +170,8 @@ Restart the app and ask the seat something. The reload runs at startup, before t ## The page (`app/`) -- **Layout**: three columns. A `FlowNavigator` rail over mailboxes, seats and the assistant's conversations; the stream; and a standing panel with the roster and the mailbox boards (plus artifacts in build mode). Below `lg` the panel opens from the header, and below `sm` the rail does too. A row's buttons in the rail show when you hover or focus the row. -- **Mailboxes**: Open `support.help` in the rail to read it and post. Your post calls the mailbox's own `post` action, the same one `fsdev run` and the [HTTP call above](#posting-to-the-mailbox-over-http) reach, and appears as `devuser`, the one user this app runs as. Lines other requests post appear while the view is open, and the view shows which specialist is working. +- **Layout**: three columns. A `FlowNavigator` rail over your conversations with the coordinator, with workers, and with the assistant; the stream; and a standing panel with the roster (plus artifacts in build mode). Below `lg` the panel opens from the header, and below `sm` the rail does too. A row's buttons in the rail show when you hover or focus the row. +- **Coordinators**: Open `support.help` in the rail to read your conversation with it and post. Your post calls the coordinator flow's `run` action, the same one the [HTTP call above](#posting-to-the-coordinator-over-http) reaches, and appears as `devuser`, the one user this app runs as. Answers appear while the view is open, and the view shows which specialist is working. - **Seats**: Open a specialist in the rail to see its kind and what it handles, under its row. The **+** on its row starts a conversation with it; your message and its reply stay in that conversation. See [Talking to one specialist](#talking-to-one-specialist). - **Roster**: The team panel's roster lists seats [hired while the app runs](#hiring-while-the-app-runs), not the four declared in files. Until you hire one, it says no seats have been hired in this organization yet, while the rail lists the four specialists. - **Session management**: Open the assistant in the rail and press the **+** on its row (New session) to start a session. Its sessions are listed under it; pick one to switch to it. @@ -311,7 +300,7 @@ apps/kitchen-sink/ rich-text-component/ Text transformations for the artifact editor weekly-digest/ Scheduled actions workforce-admin/ The operator's hire and fire actions - workforce/ The support team: teams/, the escalate tool under blocks/, workforce.gen.ts + workforce/ The support team: teams/, workforce.gen.ts components/ flow-state/ Shared item-renderer UI (installed from @flow-state-dev/ui) chat-agent/ chat-agent-specific renderers diff --git a/apps/kitchen-sink/app/page.tsx b/apps/kitchen-sink/app/page.tsx index 422efa52c6..b7107f263b 100644 --- a/apps/kitchen-sink/app/page.tsx +++ b/apps/kitchen-sink/app/page.tsx @@ -16,7 +16,7 @@ import { type FlowNavigatorLeafState, type FlowNavigatorSection, } from "@flow-state-dev/react"; -import { createClient, createResourceClient, createSessionClient, type ClientFetch } from "@flow-state-dev/client"; +import { createClient, createSessionClient, type ClientFetch } from "@flow-state-dev/client"; import { Button } from "@/components/ui/button"; import { Menu, MessageSquareText, Package, Plus, RotateCcw, Users, Wrench, X } from "lucide-react"; @@ -58,7 +58,7 @@ import { SessionItemsProvider } from "@/components/flow-state/session-items-cont import { ChatAgentMessage } from "@/components/chat-agent/message"; import { cn } from "@/lib/utils"; import { railSessions } from "@/lib/rail-sessions"; -import { MAILBOX_KINDS, defaultConversation, isMailboxKind, SEAT_KINDS, SHELL_FLOW_KIND } from "@/lib/workforce-shell"; +import { COORDINATOR_KINDS, defaultConversation, isCoordinatorKind, SEAT_KINDS, SHELL_FLOW_KIND } from "@/lib/workforce-shell"; import { createWorkforceClient, type RosterEntry } from "@flow-state-dev/workforce/browser"; import { KITCHEN_SINK_USER_ID } from "@/lib/kitchen-sink-principal"; import { pageGoalControl } from "@/lib/goal-control"; @@ -78,27 +78,27 @@ type MobilePanel = "chat" | "artifacts"; * Whether the rail lists the person's conversations with workers: the * sessions on the one `agent` copy every worker runs on. A conversation is * started from the roster panel's "Talk", which names its worker. Set this to - * `false` to ship the rail with mailboxes only. + * `false` to ship the rail with coordinators only. */ const SHOW_SEATS_IN_RAIL = true; /** - * The `fetch` the panels read with, and a board's live stream is sent with, so - * the stream goes out the way the reads do. This app sends no credential; a - * host that does adds it here, once, for both. + * The `fetch` the roster and the person's conversations with workers are read + * with. This app sends no credential; a host that does adds it here, once. */ const panelFetch: ClientFetch = (input, init) => fetch(input, init); /** * The rail's sections. How deep each kind goes is read off the flow's declared - * cardinality, never written here: a mailbox kind opens straight into its - * conversations, and so does `agent`, whose one copy holds every worker's. + * cardinality, never written here: the coordinator flow opens straight into + * the person's conversations with its workers, and so does `agent`, whose one + * copy holds every worker's. * * "Assistant" is this app's own chat flow, so its conversations stay one click * away. */ const RAIL_SECTIONS: readonly FlowNavigatorSection[] = [ - { label: "Mailboxes", kinds: MAILBOX_KINDS }, + { label: "Coordinators", kinds: COORDINATOR_KINDS }, ...(SHOW_SEATS_IN_RAIL ? [{ label: "Workers", kinds: SEAT_KINDS }] : []), { label: "Assistant", kinds: [SHELL_FLOW_KIND] }, ]; @@ -114,9 +114,9 @@ const RAIL_THEME = { /** * A session picked in the rail that is not one of the assistant's own. - * `address` is the flow it belongs to: the kind for a mailbox, and the worker - * flow's copy for a conversation with a worker. Continuing or retrying a - * request is routed by it. + * `address` is the worker flow's copy it belongs to (`coordinator` for a + * conversation with a coordinator, `agent` for one with a specialist). + * Continuing or retrying a request is routed by it. */ type PickedSession = { sessionId: string; kind: string; address: string }; @@ -153,9 +153,8 @@ function PageInner() { process.env.NEXT_PUBLIC_KITCHEN_SINK_TEST_MODE === "1" ? searchParams.get("e2eSession") : null; - // The goal control `no-live` opens the rail's panels and the team panel's - // boards without following their session, so a goal check can be seen to - // fail without it. + // The goal control `no-live` opens the rail's panels without following their + // session, so a goal check can be seen to fail without it. const pickedLive = pageGoalControl(searchParams) !== "no-live"; return ( @@ -235,11 +234,11 @@ function KitchenSinkApp({ e2eSessionId, pickedLive }: { e2eSessionId: string | n autoPlayTTS: ttsEnabled, }); - // A mailbox's or a seat's session, when one is picked in the rail. Its panel - // has a composer of its own, which calls the picked flow's own action on - // this session; the assistant's composer stays wired to the assistant. + // A coordinator's or a seat's session, when one is picked in the rail. Its + // panel has a composer of its own, which calls the picked flow's own action + // on this session; the assistant's composer stays wired to the assistant. // It follows the whole session (`live`), because others write here too: a - // seat answering a post, a person in another tab. The assistant's session + // delegate's answer landing, a person in another tab. The assistant's session // (`session`, above) is not live: only this person writes there, and their // own requests already stream. const pickedSession = useSession(picked?.sessionId, { @@ -257,10 +256,6 @@ function KitchenSinkApp({ e2eSessionId, pickedLive }: { e2eSessionId: string | n return async (requestId: string) => (await client.getRequestStatus(requestId)).status; }, [pickedAddress]); - // One resource client for every panel read, held stable: the panels fence - // their reads on it, so a new object each render would read as a new - // backend each render. Each board reads through its own mailbox's session. - const resourceClient = useMemo(() => createResourceClient({ baseUrl: "", fetcher: panelFetch }), []); // The person's roster and their conversations with workers. const workforce = useMemo( () => createWorkforceClient({ userId: KITCHEN_SINK_USER_ID, baseUrl: "", fetcher: panelFetch }), @@ -268,10 +263,10 @@ function KitchenSinkApp({ e2eSessionId, pickedLive }: { e2eSessionId: string | n ); const sessionClient = useMemo(() => createSessionClient({ baseUrl: "" }), []); - // What the rail lists: a mailbox kind's declared mailboxes, never one a kept - // store holds from an earlier roster. Stable, so the navigator's reads stay - // fenced on it. - const railSessionSource = useMemo(() => railSessions(sessionClient), [sessionClient]); + // What the rail lists: under the coordinator kind, the person's conversation + // with each coordinator the tree declares. Stable, so the navigator's reads + // stay fenced on it. + const railSessionSource = useMemo(() => railSessions(sessionClient, workforce), [sessionClient, workforce]); const clientData = useClientData(session, CLIENT_DATA_OPTIONS); const { items: artifactItems } = useResourceCollectionList(session, "artifacts", { limit: 50 }); @@ -403,8 +398,9 @@ function KitchenSinkApp({ e2eSessionId, pickedLive }: { e2eSessionId: string | n const railSlots = useMemo( () => ({ - // "New session" sits on the assistant's own row. Mailboxes get none; the - // boot opens them. A conversation with a worker starts from the roster. + // "New session" sits on the assistant's own row. Coordinators get none: + // the rail opens the person's one conversation with each. A conversation + // with a worker starts from the roster. leafToolbar: (leaf: FlowNavigatorLeafState) => leaf.kind === SHELL_FLOW_KIND ? ( setIsTeamSheetOpen(true)} > @@ -610,7 +604,7 @@ function KitchenSinkApp({ e2eSessionId, pickedLive }: { e2eSessionId: string | n {/* The model named here is the assistant's own preference, and only the - assistant answers with it. A picked mailbox or seat answers through + assistant answers with it. A picked coordinator or seat answers through its model intents (an `FSDEV_*` override lands there), and each reply carries the model that produced it, so the bar names none there. */} setIsTeamSheetOpen(false)} /> )} @@ -662,10 +656,10 @@ function KitchenSinkApp({ e2eSessionId, pickedLive }: { e2eSessionId: string | n )} style={teamPanelStyle} data-testid="team-panel" - aria-label="Boards and roster" + aria-label="Roster" >
-
@@ -736,8 +730,9 @@ function Rail({ return ( <>
- {/* Dispatch runs too: a seat's conversation for a mailbox is a run the - mailbox started, and is listed only when they are included. */} + {/* Dispatch runs too: a specialist's session for a coordinator's post is + a run the coordinator started, and is listed only when they are + included. */} is working` row for each run under the session - * that hasn't finished. With the session followed live (`useSession`'s - * `live`), a seat's line lands and its row clears without a reload; the panel + * Both panels show a ` is working` row for each run under the session + * that hasn't finished: on a coordinator, each delegate still working on a + * post it was handed. With the session followed live (`useSession`'s + * `live`), an answer lands and its row clears without a reload; the panel * keeps no timer or poll of its own for either. When the list could not be * read again, the rows it still has say `was working at the last check`. */ import { useEffect, useMemo, useRef, useState, type FormEvent, type ReactNode } from "react"; import type { useSession } from "@flow-state-dev/react"; -import { MAILBOX_POST_COMPONENT, type MailboxTranscriptLine } from "@flow-state-dev/workforce/browser"; import { Button } from "@/components/ui/button"; import { Textarea } from "@/components/ui/textarea"; -import { isMailboxKind, seatAskFor, wokenWorkerOf } from "@/lib/workforce-shell"; +import { COORDINATOR_ASK, delegateOfRun, isCoordinatorKind, seatAskFor } from "@/lib/workforce-shell"; type Session = ReturnType; /** Reads one request's own status from the server, by id. */ export type RequestStatusLookup = (requestId: string) => Promise; -/** Who a line names: its author claim, else the server's principal, else nobody. */ -export function lineLabel(line: Pick): string { - return line.author ?? line.principal ?? "unattributed"; +/** One line of a coordinator's conversation: who wrote it, and what it says. */ +export interface CoordinatorLine { + id: string; + /** A delegate's worker id, the person's id, or, for the coordinator's own word, the coordinator. */ + label: string; + body: string; } -/** The session's `mailbox-post` lines, in the order the session holds them. */ -function mailboxLines(items: Session["items"]): MailboxTranscriptLine[] { +/** + * The conversation's kept messages as lines, in the order the session holds + * them. A delegate's answer carries its name (`agentName`); the person's post + * is a `user` message, under `person`; anything else the coordinator said + * itself is under `coordinator`. Transient messages are not lines. + * + * @param items The session's items. + * @param person The name the person's own posts are drawn under. + */ +export function coordinatorLines(items: Session["items"], person: string): CoordinatorLine[] { return items.flatMap((item) => { - const component = item as { type: string; component?: string; data?: unknown }; - if (component.type !== "component" || component.component !== MAILBOX_POST_COMPONENT) return []; - const data = component.data as Partial | undefined; - return typeof data?.id === "string" && typeof data.body === "string" ? [data as MailboxTranscriptLine] : []; + const message = item as { + type: string; + id?: string; + role?: string; + agentName?: string; + transient?: boolean; + content?: Array<{ text?: string }>; + }; + if (message.type !== "message" || message.transient === true || typeof message.id !== "string") return []; + const body = (message.content ?? []).map((part) => part.text ?? "").join(""); + const label = message.agentName ?? (message.role === "user" ? person : "coordinator"); + return [{ id: message.id, label, body }]; }); } @@ -55,38 +74,41 @@ function mailboxLines(items: Session["items"]): MailboxTranscriptLine[] { * The picked session's panel. * * @param session The picked session, as `useSession` returns it. - * @param kind The picked flow's kind: a mailbox kind or a seat kind. + * @param kind The picked flow's kind: a coordinator kind or a seat kind. + * @param person The name the person's own posts are drawn under, on a coordinator. * @param requestStatus Reads a sent request's own status, so a send settles * on its request and not on whatever the session's stream did. - * @param conversation The session's item stream, drawn for a seat. A mailbox - * draws its transcript instead, so the page need not build one for it. + * @param conversation The session's item stream, drawn for a seat. A + * coordinator draws its lines instead, so the page need not build one for it. */ export function PickedSessionPanel({ session, kind, + person, requestStatus, conversation, }: { session: Session; kind: string; + person: string; requestStatus: RequestStatusLookup; conversation?: ReactNode; }) { // Each composer is keyed by the session it talks to, so a draft, a pending // send or a failure never carries over to the next pick. const composerKey = session.sessionId ?? "none"; - if (isMailboxKind(kind)) { + if (isCoordinatorKind(kind)) { return (
- + session.sendAction("post", { body: text })} + label="Post to this coordinator" + placeholder="Write a post for the team…" + send={(text) => session.sendAction(COORDINATOR_ASK.action, { [COORDINATOR_ASK.field]: text })} />
); @@ -115,18 +137,18 @@ export function PickedSessionPanel({ ); } -/** A mailbox's transcript, oldest first, each line under who it names. */ -function MailboxTranscript({ session }: { session: Session }) { - const lines = useMemo(() => mailboxLines(session.items), [session.items]); +/** A coordinator's conversation, oldest first, each line under who wrote it. */ +function CoordinatorTranscript({ session, person }: { session: Session; person: string }) { + const lines = useMemo(() => coordinatorLines(session.items, person), [session.items, person]); return ( -
+
    {lines.map((line) => ( -
  1. - - {lineLabel(line)} +
  2. + + {line.label} - + {line.body}
  3. @@ -140,10 +162,10 @@ function MailboxTranscript({ session }: { session: Session }) { } /** - * One row per run under the session that hasn't finished, named by the worker - * a mailbox woke for it, or else by the run's flow; a run with no recorded - * flow is background work, not a seat. A run - * waiting on an approval reads as working until it ends. + * One row per run under the session that hasn't finished, named by the + * delegate a coordinator handed a post to, or else by the run's flow; a run + * with no recorded flow is background work, not a seat. A run waiting on an + * approval reads as working until it ends. * * While the list is stale (`childSessionsStale`: the last re-read failed, and * the rows are the last list the hook had), a row says the run was working at @@ -159,7 +181,7 @@ function WorkingRows({ session }: { session: Session }) {
  4. {run.flowId === undefined ? `Background work ${stale ? "was running at the last check" : "is running"}` - : `${wokenWorkerOf(run) ?? run.flowId} ${stale ? "was working at the last check" : "is working"}`} + : `${delegateOfRun(run) ?? run.flowId} ${stale ? "was working at the last check" : "is working"}`}
  5. ))} diff --git a/apps/kitchen-sink/components/team-panel.tsx b/apps/kitchen-sink/components/team-panel.tsx index 36eb916ef2..a1939ab008 100644 --- a/apps/kitchen-sink/components/team-panel.tsx +++ b/apps/kitchen-sink/components/team-panel.tsx @@ -1,77 +1,37 @@ "use client"; /** - * The right-hand panel: the person's workers, and the mailbox's boards. + * The right-hand panel: the person's workers. * - * The workers are the person's roster: the specialists the app's files - * declare, which everyone has, and any worker the person hired or forked, - * each with the flow it runs on. "Talk" opens the person's conversation with - * that worker, or starts one, through Workforce's client. - * - * Each board reads through its own mailbox's session. + * The workers are the person's roster: the coordinator and the specialists the + * app's files declare, which everyone has, and any worker the person hired or + * forked, each with the flow it runs on. "Talk" opens the person's + * conversation with that worker, or starts one, through Workforce's client. */ -import { useEffect, useState, type CSSProperties, type ReactNode } from "react"; -import type { ClientFetch } from "@flow-state-dev/client"; -import { BoardList, type PanelRowSource } from "@flow-state-dev/react"; +import { useEffect, useState, type ReactNode } from "react"; import type { RosterEntry, WorkforceClient } from "@flow-state-dev/workforce/browser"; -import { SHELL_BOARDS } from "@/lib/workforce-shell"; - -const PANEL_THEME = { - "--fsd-panel-fg": "var(--color-foreground)", - "--fsd-panel-muted-fg": "var(--color-muted-foreground)", -} as CSSProperties; - export interface TeamPanelProps { /** The person's workforce client. Pass a stable one: the roster is read again only when it changes. */ workforce: WorkforceClient; /** Open the person's conversation with a worker. */ onTalk: (worker: RosterEntry) => Promise; - /** The host's resource client. Pass a stable one — the panels fence on it. */ - resourceClient: PanelRowSource; - /** - * The `fetch` `resourceClient` was built with, so a board's live stream goes - * out the way its reads do. Stable, like the client. - */ - fetcher: ClientFetch; - /** Read each board again whenever its mailbox keeps a change to it. Off unless asked. */ - live?: boolean; /** Anything the host puts above the roster, such as build mode's artifacts. */ top?: ReactNode; } -export function TeamPanel({ workforce, onTalk, resourceClient, fetcher, live = false, top }: TeamPanelProps) { +export function TeamPanel({ workforce, onTalk, top }: TeamPanelProps) { return ( -
    +
    {top}

    Workers

    - {SHELL_BOARDS.map((board) => ( -
    -

    - {board.board} - · {board.mailboxId} -

    - -
    - ))}
    ); } -function noCasesYet() { - return

    No cases filed here yet.

    ; -} - /** The person's roster, each worker with its flow, what it handles, and "Talk". */ function WorkerRoster({ workforce, onTalk }: { workforce: WorkforceClient; onTalk: (worker: RosterEntry) => Promise }) { const roster = useRoster(workforce); diff --git a/apps/kitchen-sink/e2e/talk-from-page.spec.ts b/apps/kitchen-sink/e2e/talk-from-page.spec.ts index 82a6fca771..a3a178af96 100644 --- a/apps/kitchen-sink/e2e/talk-from-page.spec.ts +++ b/apps/kitchen-sink/e2e/talk-from-page.spec.ts @@ -1,30 +1,34 @@ /** - * Talking from the page: posting to the support mailbox and messaging a + * Talking from the page: posting to the support coordinator and messaging a * specialist, from the panel the rail opens, against the built app on its * scripted model. * - * These write to `support.help`, a mailbox every scenario on this user shares, - * which is why they live here and not in `workforce-shell.spec.ts` (whose - * scenarios write to no mailbox). Each one asserts on a token of its own, so a - * line another test posted can never be the one that passes it. + * These write to the person's conversation with `support.help`, which every + * scenario on this user shares, which is why they live here and not in + * `workforce-shell.spec.ts` (whose scenarios post to no coordinator). Each one + * asserts on a token of its own, so a line another test posted can never be + * the one that passes it. * - * The mailbox is routed: each post from a person goes to one specialist, and a - * post sent while that specialist has not answered the last one goes to it - * again, with no model call. So these run one at a time, in order, and each - * waits for its last post's answer to land before it ends: a post left - * unanswered would hold the next test's post on the wrong specialist. + * The coordinator routes by best fit: each post from a person goes to one + * specialist, and a post sent while that specialist has not answered the last + * one goes to it again, with no model call. So these run one at a time, in + * order, and each waits for its last post's answer to land before it ends: a + * post left unanswered would hold the next test's post on the wrong + * specialist. * * What is asserted is always read back from the server after a reload: the * panel draws no optimistic copy, and the reload throws away anything the * page kept for itself. * - * Checks, by the spec's ids (`specs/issues/FIX-1611/BUSINESS-RULES.md`): - * BR-4 (a post runs the routed specialist once and nobody else; its answer - * lands as one line under its name), BR-6 (an answer through `post-to-mailbox` - * is still one line, and wakes nobody), BR-7 (a routed turn sees the lines - * another specialist answered), BR-8 (a case that needs a person is one row on - * `escalations`), BR-13 (a new conversation on a specialist keeps both sides - * across a reload). + * Checks, by the spec's ids (`specs/issues/FIX-1611/BUSINESS-RULES.md`), + * moved onto the coordinator by FIX-1792 (BR-8): BR-4 (a post runs the routed + * specialist once and nobody else; its answer lands as one line under its + * name), BR-6 (the answer is one line, and wakes nobody), BR-13 (a new + * conversation on a specialist keeps both sides across a reload). BR-7 (a + * routed turn sees the lines another specialist answered) has no subject on + * the coordinator, which hands a specialist the post alone; BR-8 (a case that + * needs a person is one row on `escalations`) left with the escalation + * feature. * * And, re-pointed at this roster, by `specs/issues/FIX-1585/BUSINESS-RULES.md`: * BR-1 and BR-2 (a line labelled `devuser` survives a reload), BR-17 (a failed @@ -35,17 +39,12 @@ * never reaches the page. * * And by `specs/issues/FIX-1609/BUSINESS-RULES.md`, re-pointed at this - * roster: BR-1, BR-7 and BR-8 (with the mailbox open, the routed specialist - * shows as working, then its line lands and the row clears, with no - * reload), BR-20 and BR-21 (the panel is the only view that follows its - * session: one session stream, for `support.help`, and none for the + * roster: BR-1, BR-7 and BR-8 (with the conversation open, the routed + * specialist shows as working, then its line lands and the row clears, with + * no reload), BR-20 and BR-21 (the panel is the only view that follows its + * session: one session stream, for the conversation, and none for the * assistant). That one is read before any reload, on purpose: what it * checks is the page keeping up by itself. - * - * And by `specs/issues/FIX-1622/BUSINESS-RULES.md`: BR-1 and BR-2 (a case - * filed while the page is open shows on escalations with no reload, as one - * row with its status word, and no status columns), BR-21 (each row is still - * an `li[data-task-id]` in the board's panel). Also read before any reload. */ import type { Locator, Page } from "@playwright/test"; import { test, expect } from "./fixtures"; @@ -62,7 +61,8 @@ import { test, expect } from "./fixtures"; // One at a time, in file order: see the header. test.describe.configure({ mode: "default" }); -const MAILBOX = "support.help"; +/** The coordinator the person posts to. */ +const COORDINATOR = "support.help"; const SPECIALISTS = ["support.devices", "support.accounts", "support.fsd", "support.general"] as const; const rail = (page: Page) => page.getByTestId("rail"); @@ -106,11 +106,11 @@ type WorkerSession = { id: string; parentSessionId?: string | null }; /** * The person's sessions with `worker`, as the server lists them by the - * session's readonly `workerId`; with `runs`, the conversations a mailbox + * session's readonly `workerId`; with `runs`, the sessions a coordinator * started for it too. Not graded on its own: what each holds is read on the page. */ -async function sessionsOf(page: Page, worker: string, runs = false): Promise { - const query = `flowId=${WORKER_FLOW}&userId=devuser&state.workerId=${encodeURIComponent(worker)}&limit=100${runs ? "&include=dispatch-runs" : ""}`; +async function sessionsOf(page: Page, worker: string, runs = false, flow: string = WORKER_FLOW): Promise { + const query = `flowId=${flow}&userId=devuser&state.workerId=${encodeURIComponent(worker)}&limit=100${runs ? "&include=dispatch-runs" : ""}`; const res = await page.request.get(`/api/flows/sessions?${query}`); expect(res.ok(), await res.text()).toBe(true); return ((await res.json()) as { sessions: WorkerSession[] }).sessions; @@ -118,30 +118,39 @@ async function sessionsOf(page: Page, worker: string, runs = false): Promise `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`; -/** Open the mailbox's panel. */ -async function openMailbox(page: Page): Promise { +/** The person's conversation with the coordinator: the session the rail opens. */ +async function conversationId(page: Page): Promise { + const sessions = await sessionsOf(page, COORDINATOR, false, "coordinator"); + expect(sessions.length, `the person's conversations with ${COORDINATOR}`).toBe(1); + return sessions[0]!.id; +} + +/** Open the person's conversation with the coordinator in the panel. */ +async function openHelpDesk(page: Page): Promise { await openShell(page); - await open(page, "mailbox"); - await row(page, MAILBOX).click(); + await open(page, "coordinator"); + await row(page, COORDINATOR).click(); const panel = picked(page); - await expect(panel.getByTestId("mailbox-transcript")).toBeVisible(); + await expect(panel.getByTestId("coordinator-transcript")).toBeVisible(); return panel; } -/** Post a line from the mailbox's panel, and wait until the mailbox keeps it. */ +/** Post a line from the conversation's panel, and wait until the conversation keeps it. */ async function post(panel: Locator, line: string): Promise { - await panel.getByLabel("Post to this mailbox").fill(line); + await panel.getByLabel("Post to this coordinator").fill(line); await panel.getByRole("button", { name: "Send" }).click(); - await expect(panel.getByTestId("mailbox-line").filter({ hasText: line })).toHaveCount(1); + await expect(panel.getByTestId("coordinator-line").filter({ hasText: line })).toHaveCount(1); } -type Line = { principal: string; author?: string; body: string }; +type Line = { role?: string; agentName?: string; text: string }; -/** The mailbox's lines as the server keeps them, oldest first. Not graded: used only to wait. */ +/** The conversation's lines as the server keeps them, oldest first. Not graded: used only to wait. */ async function keptLines(page: Page): Promise { - const res = await page.request.get(`/api/flows/sessions/${MAILBOX}/state?include_items=true&item_types=component&limit=1000`); - const items = ((await res.json()) as { items?: Array<{ component?: string; data?: Line }> }).items ?? []; - return items.filter((item) => item.component === "mailbox-post").map((item) => item.data!); + const res = await page.request.get(`/api/flows/sessions/${await conversationId(page)}/state?include_items=true&item_types=message&limit=1000`); + const items = ((await res.json()) as { items?: Array<{ role?: string; agentName?: string; transient?: boolean; content?: Array<{ text?: string }> }> }).items ?? []; + return items + .filter((item) => item.transient !== true) + .map((item) => ({ role: item.role, agentName: item.agentName, text: (item.content ?? []).map((c) => c.text ?? "").join("") })); } /** @@ -154,27 +163,27 @@ async function answerLanded(page: Page, mark: string): Promise { await expect .poll(async () => { const lines = await keptLines(page); - const at = lines.findIndex((l) => l.author === undefined && l.body.includes(mark)); - return at !== -1 && lines.slice(at + 1).some((l) => l.author !== undefined); + const at = lines.findIndex((l) => l.role === "user" && l.text.includes(mark)); + return at !== -1 && lines.slice(at + 1).some((l) => l.agentName !== undefined); }, { timeout: 20_000 }) .toBe(true); } /** - * The mailbox as drawn after a reload: each line's label and text, oldest - * first. The transcript mounts before its lines load, so this waits for the - * person's line carrying `mark` first. + * The conversation as drawn after a reload: each line's label and text, + * oldest first. The transcript mounts before its lines load, so this waits + * for the person's line carrying `mark` first. */ async function drawnAfterReload(page: Page, mark: string): Promise> { await page.reload(); - const panel = await openMailbox(page); - const person = page.getByTestId("mailbox-line-label").getByText("devuser", { exact: true }); - await expect(panel.getByTestId("mailbox-line").filter({ has: person }).filter({ hasText: mark }).first()).toBeVisible(); + const panel = await openHelpDesk(page); + const person = page.getByTestId("coordinator-line-label").getByText("devuser", { exact: true }); + await expect(panel.getByTestId("coordinator-line").filter({ has: person }).filter({ hasText: mark }).first()).toBeVisible(); return await panel - .getByTestId("mailbox-line") + .getByTestId("coordinator-line") .evaluateAll((els) => els.map((el) => ({ - label: el.querySelector('[data-testid="mailbox-line-label"]')?.textContent ?? "", + label: el.querySelector('[data-testid="coordinator-line-label"]')?.textContent ?? "", text: el.textContent ?? "", })), ); @@ -190,27 +199,28 @@ function answersTo(drawn: Array<{ label: string; text: string }>, mark: string) } /** - * `seat`'s conversation for the mailbox, opened and drawn, or `undefined` when - * it has none. A worker keeps one conversation per mailbox: the run the - * mailbox started, created naming the worker. + * `seat`'s session for the person's conversation with the coordinator, opened + * and drawn, or `undefined` when it has none. A delegate keeps one session per + * conversation: the run the conversation started, created naming the worker. * * Reloads first, so nothing is picked, then waits for the copy's list to load - * and the conversation to draw its first turn: a turn counted as missing is - * read off a loaded conversation, never off one still fetching. + * and the session to draw its first turn: a turn counted as missing is read + * off a loaded session, never off one still fetching. */ -async function mailboxConversationOf(page: Page, seat: string): Promise { +async function delegateSessionOf(page: Page, seat: string): Promise { await page.reload(); await expect(page.locator('[data-testid="message-input"]:visible')).toBeEnabled(); - const runs = (await sessionsOf(page, seat, true)).filter((s) => s.parentSessionId === MAILBOX); - expect(runs.length, `${seat} has ${runs.length} conversations for ${MAILBOX}`).toBeLessThanOrEqual(1); + const parent = await conversationId(page); + const runs = (await sessionsOf(page, seat, true)).filter((s) => s.parentSessionId === parent); + expect(runs.length, `${seat} has ${runs.length} sessions for the conversation with ${COORDINATOR}`).toBeLessThanOrEqual(1); if (runs.length === 0) return undefined; const leaf = await openWorkerCopy(page); const run = leaf.locator(`[data-session-id="${runs[0]!.id}"]`); - // Listed as the run the mailbox started. - await expect(run).toHaveAttribute("data-dispatch-run-of", MAILBOX); + // Listed as the run the conversation started. + await expect(run).toHaveAttribute("data-dispatch-run-of", parent); await run.click(); const conversation = picked(page); - // A conversation for the mailbox opens on a person's post. + // A delegate's session opens on a person's post. await expect(conversation.locator('[data-message-role="user"]').first()).toBeVisible(); return conversation; } @@ -220,19 +230,19 @@ test("a line posted to support.help shows as devuser, and is still there after a consoleErrors: _consoleErrors, }) => { const line = `help line ${token()}`; - const panel = await openMailbox(page); + const panel = await openHelpDesk(page); await expect(page.getByText(/go to the assistant/)).toHaveCount(0); await post(panel, line); - const posted = panel.getByTestId("mailbox-line").filter({ hasText: line }); - await expect(posted.getByTestId("mailbox-line-label")).toHaveText("devuser"); - await expect(panel.getByLabel("Post to this mailbox")).toHaveValue(""); + const posted = panel.getByTestId("coordinator-line").filter({ hasText: line }); + await expect(posted.getByTestId("coordinator-line-label")).toHaveText("devuser"); + await expect(panel.getByLabel("Post to this coordinator")).toHaveValue(""); await answerLanded(page, line); await page.reload(); - const kept = (await openMailbox(page)).getByTestId("mailbox-line").filter({ hasText: line }); + const kept = (await openHelpDesk(page)).getByTestId("coordinator-line").filter({ hasText: line }); await expect(kept).toHaveCount(1); - await expect(kept.getByTestId("mailbox-line-label")).toHaveText("devuser"); + await expect(kept.getByTestId("coordinator-line-label")).toHaveText("devuser"); }); test("a new support.devices conversation keeps the message and the reply across a reload", async ({ @@ -303,7 +313,7 @@ test("a post to support.help gets one answer, as a line under the specialist it }) => { const mark = `answer-token-${token()}`; const line = `[route:support.accounts] [scenario:wake] ${mark} where is my refund?`; - await post(await openMailbox(page), line); + await post(await openHelpDesk(page), line); await answerLanded(page, mark); const answers = answersTo(await drawnAfterReload(page, mark), mark); @@ -313,77 +323,33 @@ test("a post to support.help gets one answer, as a line under the specialist it expect(answers[0]!.text).not.toContain(mark); }); -test("a post that needs a person is one row on escalations in the team panel after a reload, and the line says so", async ({ - page, - consoleErrors: _consoleErrors, -}) => { - const mark = `case-token-${token()}`; - await post(await openMailbox(page), `[route:support.devices] [scenario:needs-a-person] ${mark} the charger caught fire`); - await answerLanded(page, mark); - // The row is written by the mailbox's own request, a moment after the - // dispatch. Let it land before the reload; nothing here is graded. - await expect - .poll(async () => - (await page.request.get(`/api/flows/sessions/${MAILBOX}/resources/${MAILBOX}.escalations`)).text(), - ) - .toContain(mark); - - const answers = answersTo(await drawnAfterReload(page, mark), mark); - expect(answers.map((l) => l.label)).toEqual(["support.devices"]); - expect(answers[0]!.text).toContain("[reply:escalated]"); - const board = page.getByTestId(`board-${MAILBOX}.escalations`); - await expect(board.locator("li[data-task-id]").filter({ hasText: mark })).toHaveCount(1); -}); - -test("a post that needs a person shows on escalations in the team panel with no reload, as one row in a list with its status", async ({ - page, - consoleErrors: _consoleErrors, -}) => { - const mark = `case-token-${token()}`; - const mailbox = await openMailbox(page); - // The board is read before the post, so a row that shows afterwards came - // from the page keeping up by itself, not from that read. - const board = page.getByTestId(`board-${MAILBOX}.escalations`); - await expect(board.locator('[data-panel="board"]')).toBeVisible(); - await expect(board.locator('[data-state="loading"]')).toHaveCount(0); - - await post(mailbox, `[route:support.devices] [scenario:needs-a-person] ${mark} the charger caught fire`); - - const filed = board.locator("li[data-task-id]").filter({ hasText: mark }); - await expect(filed).toHaveCount(1, { timeout: 15_000 }); - await expect(filed.locator("[data-task-status]")).toHaveText("pending"); - // One list, not a column per status. - await expect(board.locator("[data-column]")).toHaveCount(0); - await answerLanded(page, mark); -}); - -test("a post to support.help runs the specialist it was routed to once, in its own conversation for the mailbox, and nobody else", async ({ +test("a post to support.help runs the specialist it was routed to once, in its own session for the conversation, and nobody else", async ({ page, consoleErrors: _consoleErrors, }) => { const first = `[route:support.fsd] [scenario:wake] wake ${token()} how do I stream a generator?`; const second = `[route:support.fsd] [scenario:wake] wake ${token()} and cancel one?`; - const mailbox = await openMailbox(page); + const panel = await openHelpDesk(page); for (const line of [first, second]) { - await post(mailbox, line); + await post(panel, line); await answerLanded(page, line); } // Time for a wrongly woken seat to run, so its absence below is not a race. await page.waitForTimeout(1_500); - // One conversation for the mailbox, whatever else the worker holds. - const conversation = await mailboxConversationOf(page, "support.fsd"); - expect(conversation, `support.fsd has no conversation for ${MAILBOX}`).toBeDefined(); + // One session for the conversation, whatever else the worker holds. + const conversation = await delegateSessionOf(page, "support.fsd"); + expect(conversation, `support.fsd has no session for the conversation with ${COORDINATOR}`).toBeDefined(); for (const line of [first, second]) { const heard = conversation!.locator('[data-message-role="user"]').filter({ hasText: line }); await expect(heard).toHaveCount(1); - await expect(heard).toContainText(`in ${MAILBOX}: ${line}`); + await expect(heard).toContainText(`through ${COORDINATOR}: ${line}`); } await expect(conversation!.locator('[data-message-role="assistant"]').filter({ hasText: "[reply:wake]" })).not.toHaveCount(0); // The other specialists never heard either post. for (const seat of SPECIALISTS.filter((s) => s !== "support.fsd")) { - const other = await mailboxConversationOf(page, seat); + const other = await delegateSessionOf(page, seat); if (other === undefined) continue; for (const line of [first, second]) { await expect(other.locator('[data-message-role="user"]').filter({ hasText: line })).toHaveCount(0); @@ -391,57 +357,36 @@ test("a post to support.help runs the specialist it was routed to once, in its o } }); -test("a specialist that answers through post-to-mailbox has one line under its own name, and the line wakes nobody", async ({ +test("a specialist's answer is one line under its own name, and the line wakes nobody", async ({ page, consoleErrors: _consoleErrors, }) => { const mark = `reply-token-${token()}`; const line = `[route:support.devices] [scenario:reply-in-mailbox] ${mark} when do refunds post?`; - await post(await openMailbox(page), line); + await post(await openHelpDesk(page), line); await answerLanded(page, mark); // Time for a wrongly woken seat to run, so its absence below is not a race. await page.waitForTimeout(1_500); const answers = answersTo(await drawnAfterReload(page, mark), mark); - // The tool's line is the answer: one line, not the tool's and the reply's. + // One line, the specialist's answer. expect(answers.map((l) => l.label)).toEqual(["support.devices"]); expect(answers[0]!.text).toContain(`[reply:in-mailbox] ${mark}`); for (const seat of SPECIALISTS) { const routed = seat === "support.devices"; - const conversation = await mailboxConversationOf(page, seat); + const conversation = await delegateSessionOf(page, seat); if (conversation === undefined) { - expect(routed, `${seat} has no conversation for ${MAILBOX}`).toBe(false); + expect(routed, `${seat} has no session for the conversation with ${COORDINATOR}`).toBe(false); continue; } const turns = conversation.locator('[data-message-role="user"]').filter({ hasText: mark }); // The routed specialist heard the person's post once; its own line reached no seat. await expect(turns, `${seat}'s turns carrying ${mark}`).toHaveCount(routed ? 1 : 0); - if (routed) await expect(turns).toContainText(`devuser in ${MAILBOX}: ${line}`); + if (routed) await expect(turns).toContainText(`devuser, through ${COORDINATOR}: ${line}`); } }); -test("a post routed to one specialist names a token from the line another specialist answered", async ({ - page, - consoleErrors: _consoleErrors, -}) => { - const mark = `reply-token-${token()}`; - const earlier = `[route:support.devices] [scenario:reply-in-mailbox] ${mark} my laptop will not charge`; - const mailbox = await openMailbox(page); - await post(mailbox, earlier); - await answerLanded(page, mark); - const ask = `[route:support.accounts] [scenario:recall] recall ${token()} what did devices say?`; - await post(mailbox, ask); - await answerLanded(page, ask); - - // support.accounts was never sent the earlier post: the token reached it - // only as one of the mailbox's recent lines. - const answers = answersTo(await drawnAfterReload(page, ask), ask); - expect(answers.map((l) => l.label)).toEqual(["support.accounts"]); - expect(answers[0]!.text).toContain("[reply:recall]"); - expect(answers[0]!.text).toContain(mark); -}); - test("with support.help open, the routed specialist shows as working and then its line lands, with no reload", async ({ page, consoleErrors: _consoleErrors, @@ -455,22 +400,22 @@ test("with support.help open, the routed specialist shows as working and then it const mark = `reply-token-${token()}`; const line = `[route:support.devices] [scenario:reply-after-a-hold] ${mark} when do refunds post?`; - const mailbox = await openMailbox(page); - await post(mailbox, line); + const panel = await openHelpDesk(page); + await post(panel, line); // The routed specialist holds its answer about three seconds: long enough to // be seen working. It is the only one working on the post. - const working = mailbox.getByTestId("working-row"); + const working = panel.getByTestId("working-row"); await expect(working).toHaveText(["support.devices is working"]); - const reply = mailbox.getByTestId("mailbox-line").filter({ hasText: `[reply:in-mailbox] ${mark}` }); + const reply = panel.getByTestId("coordinator-line").filter({ hasText: `[reply:in-mailbox] ${mark}` }); await expect(reply).toHaveCount(0); // Then its line lands in the open panel, under its name, and the row clears. await expect(reply).toHaveCount(1, { timeout: 15_000 }); - await expect(reply.getByTestId("mailbox-line-label")).toHaveText("support.devices"); + await expect(reply.getByTestId("coordinator-line-label")).toHaveText("support.devices"); await expect(working).toHaveCount(0); - // Only the mailbox's session is followed, by its panel and by the team - // panel's board; the assistant's is not. - expect([...new Set(followed)]).toEqual([MAILBOX]); + // Only the conversation's session is followed, by its panel; the + // assistant's is not. + expect([...new Set(followed)]).toEqual([await conversationId(page)]); }); diff --git a/apps/kitchen-sink/e2e/workforce-shell.spec.ts b/apps/kitchen-sink/e2e/workforce-shell.spec.ts index def5e502ca..f2ab9714d4 100644 --- a/apps/kitchen-sink/e2e/workforce-shell.spec.ts +++ b/apps/kitchen-sink/e2e/workforce-shell.spec.ts @@ -1,20 +1,19 @@ /** - * The rebuilt shell, against the built app: the rail browses mailboxes and - * the worker flow to the depth each flow declares, the roster panel lists each - * worker with its flow and what it handles, the panel stands beside the + * The rebuilt shell, against the built app: the rail browses the coordinator + * and the worker flow to the depth each flow declares, the roster panel lists + * each worker with its flow and what it handles, the panel stands beside the * stream, and the three regions give way in the right order as the window * narrows. * * Every worker runs on one copy of the flow it names, so the rail's worker * section is the `agent` kind, its one copy, and that copy's conversations: - * each one a session that names its worker when it is created. + * each one a session that names its worker when it is created. The + * coordinator flow is a singleton: its row opens straight into the person's + * conversation with each coordinator the tree declares. * - * These open the app as `devuser`, not a per-test user, because that is who - * the boot opens the mailboxes for: a fresh user would see no mailbox - * conversations at all. Nothing here writes to a mailbox, so sharing the user - * across parallel tests is safe. The first test does store a session for a - * mailbox the tree no longer declares, as a store kept across an upgrade - * would; the rail never draws it, and no scenario posts to it. + * These open the app as `devuser`, the one user this app runs as. Nothing + * here posts to a coordinator, so sharing the user across parallel tests is + * safe. * * The network half of the first test is what makes it more than a picture. * Fetch sessions for every row up front and every DOM assertion still passes; @@ -32,8 +31,8 @@ import { test, expect, openKitchenSink } from "./fixtures"; const SEAT = "support.devices"; /** The flow every worker in this app runs on, registered as one copy at its kind. */ const WORKER_FLOW = "agent"; -/** A mailbox the old roster declared and this one does not. */ -const RETIRED_MAILBOX = "support.desk"; +/** The coordinator the app's workforce tree declares. */ +const COORDINATOR = "support.help"; /** Session-list requests, recorded from the moment this is called. */ function sessionListRequests(page: Page): { take: () => string[] } { @@ -55,18 +54,6 @@ function requestsMatching(page: Page, pattern: RegExp): { take: () => string[] } }; } -/** - * What a store kept across an upgrade still holds from the old roster: a - * session of the mailbox kind for a mailbox the tree no longer declares. - */ -async function seedRetiredMailbox(page: Page): Promise { - const response = await page.request.post(`/api/flows/mailbox/sessions`, { - data: { userId: "devuser", sessionId: RETIRED_MAILBOX }, - }); - // Already there from an earlier scenario on this server is as good. - expect([200, 201, 409], await response.text()).toContain(response.status()); -} - /** * Give the worker one conversation: a session of its flow's one copy, created * naming it, so the copy has something to open into. @@ -92,38 +79,34 @@ const kindRow = (page: Page, kind: string) => rail(page).locator(`button[data-ki /** A copy's row under its kind. */ const copyRow = (page: Page, id: string) => rail(page).locator(`button[data-instance-id="${id}"]`); -test("the rail opens a mailbox kind into conversations and the worker flow into its one copy, reading only leaves", async ({ +test("the rail opens the coordinator into the person's conversation with it and the worker flow into its one copy, reading only leaves", async ({ page, consoleErrors: _consoleErrors, }) => { const seatSessionId = await seedSeatSession(page); - await seedRetiredMailbox(page); const requests = sessionListRequests(page); await openShell(page); - await expect(rail(page).getByRole("list", { name: "Mailboxes" })).toBeVisible(); + await expect(rail(page).getByRole("list", { name: "Coordinators" })).toBeVisible(); await expect(rail(page).getByRole("list", { name: "Workers" })).toBeVisible(); // Drawing the rail reads no session list of its own. The reads at load are // the assistant's, for the conversation the stream opens on, and the roster - // panel's, for the person's roster session. The team panel's live board - // holds its mailbox's stream open, so the page never goes network-idle: - // wait for the board's first read, then the half-second quiet that - // network-idle means. - await expect(page.getByTestId("board-support.help.escalations").locator('[data-panel="board"]')).toBeVisible(); - await page.waitForTimeout(500); + // panel's, for the person's roster session. + await page.waitForLoadState("networkidle"); expect( requests.take().filter((url) => !url.includes("flowKind=chat-agent") && !url.includes("flowKind=workforce-roster")), ).toEqual([]); - // A mailbox kind is a singleton: its row is the leaf, so opening it lands - // straight on the mailbox conversations. It is ONE read, of the mailbox the - // tree declares, by id: the kind is never listed, so a mailbox the store - // still holds from the old roster is never drawn. - const mailboxReads = requestsMatching(page, /\/api\/flows\/sessions\/support\.help$/); - await row(page, "mailbox").click(); - await expect(row(page, "support.help")).toBeVisible(); - await expect(row(page, RETIRED_MAILBOX)).toHaveCount(0); - expect(mailboxReads.take()).toHaveLength(1); - expect(requests.take()).toEqual([]); + // The coordinator flow is a singleton: its row is the leaf, so opening it + // lands straight on the person's conversation with each coordinator the + // tree declares, found by the coordinator's id. The kind is never listed + // whole: every list it reads names the coordinator's worker. + await row(page, "coordinator").click(); + await expect(row(page, COORDINATOR)).toBeVisible(); + const coordinatorReads = requests.take().filter((url) => !url.includes("flowKind=workforce-roster")); + expect(coordinatorReads.length).toBeGreaterThan(0); + for (const url of coordinatorReads) { + expect(new URL(url).searchParams.get("state.workerId"), url).toBe(COORDINATOR); + } // The worker flow is a collection with one copy: opening its kind lists the // copy and reads nothing. No copy is drawn per worker. @@ -144,15 +127,15 @@ test("the rail opens a mailbox kind into conversations and the worker flow into /** * The header's model is the assistant's own preference, and only the assistant - * answers with it. A mailbox or a seat answers through its model intents, which - * an `FSDEV_*` override can point anywhere, and its replies carry the model - * that ran. So on those views the header names no model: one it named could - * contradict every reply beneath it. + * answers with it. A coordinator or a seat answers through its model intents, + * which an `FSDEV_*` override can point anywhere, and its replies carry the + * model that ran. So on those views the header names no model: one it named + * could contradict every reply beneath it. * * Red state produced before this was trusted: pass the assistant's model to the - * header whatever is picked, and the mailbox assertion fails (the label is still there). + * header whatever is picked, and the coordinator assertion fails (the label is still there). */ -test("the header names the assistant's model on the assistant only, never on a mailbox or a seat", async ({ +test("the header names the assistant's model on the assistant only, never on a coordinator or a seat", async ({ page, consoleErrors: _consoleErrors, }) => { @@ -165,8 +148,8 @@ test("the header names the assistant's model on the assistant only, never on a m const expand = async (button: Locator) => { if ((await button.getAttribute("aria-expanded")) !== "true") await button.click(); }; - await expand(row(page, "mailbox")); - await row(page, "support.help").click(); + await expand(row(page, "coordinator")); + await row(page, COORDINATOR).click(); await expect(page.locator('[data-testid="picked-session"]:visible')).toBeVisible(); await expect(headerModel).toHaveCount(0); @@ -608,8 +591,8 @@ test("the rail, fully expanded in both hosts, draws each action on its row, one }) => { test.setTimeout(180_000); // The worker flow's copy holding a conversation with no title and one with a - // title too long for the rail, a copy with none, and the singleton mailboxes - // the boot opens. The app draws no detail under a copy: a worker's flow and + // title too long for the rail, a copy with none, and the singleton + // coordinator flow. The app draws no detail under a copy: a worker's flow and // what it handles are on the roster panel, which the VG test below reads. await seedSeatSession(page); await seedSeatSession(page, LONG_TITLE); @@ -628,7 +611,7 @@ test("the rail, fully expanded in both hosts, draws each action on its row, one expect((await measureRows(shellNav)).scrollers).toBe(1); await checkRail(page, shellNav, "/", { hoverRow: `[data-kind="${SHELL_KIND}"]`, - quietRow: '[data-kind="mailbox"]', + quietRow: '[data-kind="coordinator"]', container: rail(page), file: "rail-kitchen-sink.png", }); @@ -719,15 +702,15 @@ for (const { width, rail: railShown, panel: panelShown } of [ // What yielded is still one tap away, from the header. if (!panelShown) { - await page.getByRole("button", { name: "Open boards and roster" }).click(); + await page.getByRole("button", { name: "Open the roster" }).click(); await expect(page.getByTestId("team-panel")).toBeVisible(); await expect(page.getByTestId("roster-panel")).toBeVisible(); - await page.getByRole("button", { name: "Close boards and roster" }).last().click(); + await page.getByRole("button", { name: "Close the roster" }).last().click(); } if (!railShown) { await page.getByRole("button", { name: "Browse" }).click(); await expect(rail(page)).toBeVisible(); - await expect(row(page, "mailbox")).toBeVisible(); + await expect(row(page, "coordinator")).toBeVisible(); } }); } diff --git a/apps/kitchen-sink/fsdev.config.ts b/apps/kitchen-sink/fsdev.config.ts index 47d73765ab..2cf895db74 100644 --- a/apps/kitchen-sink/fsdev.config.ts +++ b/apps/kitchen-sink/fsdev.config.ts @@ -19,28 +19,18 @@ import { after } from "next/server"; import path from "node:path"; import { createGateway } from "@ai-sdk/gateway"; import { createFlowState, inMemoryStores, filesystemStores, type FlowState } from "@flow-state-dev/engine"; -import { createSessionClient } from "@flow-state-dev/client"; import { OpenAIVoiceProvider } from "@flow-state-dev/voice-openai"; import { vercelPostgresStores } from "@flow-state-dev/vercel/store"; import { createScheduledTransportAdapter } from "@flow-state-dev/scheduled"; import { setScheduleIndexImpl } from "@/lib/schedule-index"; -import { - KITCHEN_SINK_ORG_ID, - KITCHEN_SINK_USER_ID, - resolveKitchenSinkPrincipal, -} from "@/lib/kitchen-sink-principal"; +import { resolveKitchenSinkPrincipal } from "@/lib/kitchen-sink-principal"; import { DEFAULT_KITCHEN_SINK_MODEL } from "@/lib/models"; import { createKitchenSinkTestModelResolver } from "@/test/mock-flowstate"; import chatAgentFlow from "@/flows/chat-agent/flow"; import richTextComponentFlow from "@/flows/rich-text-component/flow"; import weeklyDigestFlow from "@/flows/weekly-digest/flow"; import { buildKitchenSinkWorkforce } from "@/workforce/hire"; -import { - describePreRenameMarks, - findPreRenameMarks, - mergeSeatFlows, - openMailboxes, -} from "@flow-state-dev/workforce"; +import { mergeSeatFlows } from "@flow-state-dev/workforce"; import { bullmqWorker } from "@flow-state-dev/bullmq"; const gatewayApiKey = process.env.AI_GATEWAY_API_KEY; @@ -93,26 +83,15 @@ for (const failedPath of workforce.errors) { console.error(`[workforce] could not read ${failedPath}`); } -// One instance per mailbox KIND the tree selected, never one per mailbox — a -// mailbox kind is a singleton, so its address is its kind and every mailbox is -// a named session on it. -// -// These replace the hand-registered built-in this app used to carry: only the binder hands a kind the ledgers a roster minted, so -// an instance built by hand answers no board call however many `boards:` lines -// the tree declares. -const mailboxFlows = Object.fromEntries( - workforce.mailboxFlows.map((instance) => [instance.id, instance]) -); - -// The workforce's copies are addressed by their flow's kind (`agent`, and -// `workforce-roster` for the roster flow). A session with a worker is a session -// on its flow's copy, created naming the worker. `mergeSeatFlows` refuses a -// copy at an id a flow of this app already holds rather than letting it -// replace that flow. +// The workforce's copies are addressed by their flow's kind (`agent`, +// `coordinator`, and `workforce-roster` for the roster flow). A session with a +// worker is a session on its flow's copy, created naming the worker: a +// person's conversation with `support.help` is a session on `coordinator`. +// `mergeSeatFlows` refuses a copy at an id a flow of this app already holds +// rather than letting it replace that flow. const flowstate = createFlowState({ flows: mergeSeatFlows( { - ...mailboxFlows, chatAgent: chatAgentFlow, richTextComponent: richTextComponentFlow, weeklyDigest: weeklyDigestFlow, @@ -204,7 +183,7 @@ const flowstate = createFlowState({ worker: bullmqDispatch ? bullmq : undefined, adapters: [createScheduledTransportAdapter()], // Who every caller is, for every flow that brings no resolver of its own: - // the assistant's flow, the workforce's copies, every mailbox. One + // the assistant's flow and the workforce's copies. One // organization and one user, both constants, read from nothing on the // request (`lib/kitchen-sink-principal.ts`). `weekly-digest` keeps its own. resolvePrincipal: resolveKitchenSinkPrincipal, @@ -213,105 +192,6 @@ const flowstate = createFlowState({ }, }); -const runtime = await flowstate.getRuntime(); - -// --------------------------------------------------------------------------- -// The mailboxes the tree declared, opened. -// -// After `createFlowState`, not beside the workforce build above, because opening a -// mailbox is a session create and there is no session route until the -// FlowState exists. Awaited at module scope for the reason the build is: both -// the Next route handlers and the `fsdev` CLI import this module, so finishing -// here is what guarantees no request arrives before the mailboxes are open. -// -// Unguarded on every boot, deliberately. Opening is idempotent — an open -// mailbox is left exactly as it is — and the board list is the one thing -// re-opening carries, so a "first boot only" flag would strand a board added -// to a `MAILBOX.md` later. -// --------------------------------------------------------------------------- - -/** - * Who every mailbox session belongs to: the app's one user. - * - * A session belongs to one user, so a mailbox does too. The session route takes - * its owner from the resolved principal, which is `KITCHEN_SINK_USER_ID` for - * every caller, so opening the mailboxes as anyone else would make the binder - * refuse its own sessions on the next boot. - */ -const MAILBOX_OWNER = KITCHEN_SINK_USER_ID; - -// The session client, over this app's own router rather than over the network: -// the app is the server, so a loopback fetcher hands the request straight to -// the handler the Next route would have called. -const mailboxSessions = createSessionClient({ - fetcher: async (input, init) => { - const router = await flowstate.getRouter(); - // The client builds `/api/flows/...`; the catch-all handler takes the - // segments beneath that prefix as its `path` param. - const url = new URL(String(input), "http://kitchen-sink.local"); - const path = url.pathname - .replace(/^\/api\/flows\/?/, "") - .split("/") - .filter((segment) => segment.length > 0) - .map(decodeURIComponent); - const method = (init?.method ?? "GET").toUpperCase(); - if (method !== "GET" && method !== "POST" && method !== "PATCH" && method !== "DELETE") { - throw new Error(`[workforce] the mailbox session client does not issue ${method}`); - } - return await router[method](new Request(url, init), { params: { path } }); - }, -}); - -/** What to do about a store this app will not open: the same for every reason it refuses one. */ -const RESET_THE_STORE = - `Earlier data is not carried over. Delete the store and restart: for the dev profile ` + - `(STORE_TYPE=filesystem) remove .fsdev/data; for the prod profile, point FSD_DB_URL at an empty database.`; - -// A store written before this app named its organization holds its mailbox -// sessions under the framework's development organization, and this app can -// neither open nor read them. There is no upgrade path: the store is wiped and -// the app starts fresh. So the boot stops here and says so, naming every such -// mailbox, instead of failing on the first one with a bare 403 from the open -// below. It only reads: it writes, moves and deletes nothing, so two processes -// booting over the same store at once cannot race each other here. -{ - const stale: string[] = []; - for (const mailbox of workforce.mailboxes) { - const stored = await runtime.stores.session.get(mailbox.id); - // A session stored with no organization at all (BP-030) is not this app's either. - if (stored !== undefined && stored.orgId !== KITCHEN_SINK_ORG_ID) { - stale.push(`"${mailbox.id}" (organization "${stored.orgId ?? "none"}")`); - } - } - if (stale.length > 0) { - throw new Error( - `[workforce] this store was written before kitchen-sink ran as organization ` + - `"${KITCHEN_SINK_ORG_ID}", and its mailboxes belong to another organization: ` + - `${stale.join(", ")}. ${RESET_THE_STORE}`, - ); - } -} - -// A store written before mailboxes were renamed holds sessions, transcript -// lines and inventory rows under names nothing reads any more, so it is not -// carried over either. Same answer as above: stop, name each mark, say how to -// reset. Keyed on the store rather than the kind, because a custom kind kept -// its name through the rename. Reads only. -{ - const marks = await findPreRenameMarks(runtime.stores, { - mailboxIds: workforce.mailboxes.map((mailbox) => mailbox.id), - orgIds: [KITCHEN_SINK_ORG_ID], - }); - if (marks.sessions.length > 0 || marks.organizations.length > 0) { - throw new Error(`[workforce] this store was ${describePreRenameMarks(marks)}. ${RESET_THE_STORE}`); - } -} - -await openMailboxes(workforce.mailboxes, { - client: mailboxSessions, - userId: MAILBOX_OWNER, -}); - export default flowstate; // `next dev` re-evaluates this module on every HMR edit, building a fresh diff --git a/apps/kitchen-sink/lib/coordinator-control.ts b/apps/kitchen-sink/lib/coordinator-control.ts new file mode 100644 index 0000000000..3c232118b2 --- /dev/null +++ b/apps/kitchen-sink/lib/coordinator-control.ts @@ -0,0 +1,40 @@ +/** + * The goal checks' controls on `support.help`, the coordinator (`GOAL_CONTROL`). + * + * - `no-route` reads every coordinator on `routing: best-fit` as + * `routing: everyone`, so every delegate hears every post. A check that one + * specialist answers and nobody else runs must FAIL under it. + * - `answers-go-on` reads every coordinator with `rounds: 1`, so each answer + * goes back out to the other delegates. A check that an answer wakes nobody + * must FAIL under it. + * - `no-delivery` registers the coordinator flow with no flow a delegate takes + * a post on, so a post reaches nobody. A check that the routed specialist + * runs must FAIL under it. + * + * Each changes what the app hands the package, never the package: a worker's + * settings as the files are read, or the flows the coordinator is built with. + * + * Honoured only under `KITCHEN_SINK_TEST_MODE=1` (`goalControl`), so a control + * can never reach a deployed build. + */ +import type { WorkerManifest } from "@flow-state-dev/workforce"; + +import { goalControl } from "./goal-control"; + +/** The workers as read, or with each coordinator's settings changed when a control asks. */ +export function coordinatorWorkersUnderControl(workers: WorkerManifest[]): WorkerManifest[] { + const control = goalControl(); + if (control !== "no-route" && control !== "answers-go-on") return workers; + return workers.map((worker) => { + if (worker.declared.flow !== "coordinator") return worker; + if (control === "answers-go-on") return { ...worker, declared: { ...worker.declared, rounds: 1 } }; + return worker.declared.routing === "best-fit" + ? { ...worker, declared: { ...worker.declared, routing: "everyone" } } + : worker; + }); +} + +/** The flows a delegate takes a post on, or none under `no-delivery`. */ +export function delegateFlowsUnderControl(flows: T[]): T[] { + return goalControl() === "no-delivery" ? [] : flows; +} diff --git a/apps/kitchen-sink/lib/e2e-mock-script.ts b/apps/kitchen-sink/lib/e2e-mock-script.ts index 61293e40c7..b2a0180aa0 100644 --- a/apps/kitchen-sink/lib/e2e-mock-script.ts +++ b/apps/kitchen-sink/lib/e2e-mock-script.ts @@ -6,19 +6,19 @@ * assistant-generator, auto-title). Each gets its own mock instance below. * An `agent` seat answers through `agent-answer` (or * `agent-answer-with-activate-tool`, when the seat turns that tool on); both - * share `agentSeatMock`, keyed on its own scenario markers: one answers in a - * mailbox through `post-to-mailbox`, one files a case through `escalate` and - * then says so, and one names the tokens the seat could see from before the - * turn. Under the goal checks' `no-history` control the scripted model is + * share `agentSeatMock`, keyed on its own scenario markers: each answers a post + * a coordinator delivered, or a person's own turn, and one names the tokens the + * seat could see from before the turn. Under the goal checks' `no-history` control the scripted model is * handed only the system messages and the latest turn. * A scenario can hold its answer for a while first (`holdMs`), which * `test/mock-flowstate.ts` applies before the model runs. * `policy: "allow"` (set in `test/mock-flowstate.ts`) catches anything else * with a no-op model. * - * A routed mailbox's one evaluation, block name `mailbox-route`, is scripted - * by `mailboxRouteMock`: `[route:]` in a post picks that member, and - * a post that names none fails the call, so the mailbox's fallback takes it. + * A coordinator's best-fit evaluation, block name `coordinator-route`, is + * scripted by `coordinatorRouteMock`: `[route:]` in a post picks that + * delegate, and a post that names none fails the call, so the coordinator's + * fallback takes it. * * `assistantMock` uses a hand-rolled scenario dispatcher because the * built-in `mockGenerator` only supports either plain sequential steps @@ -98,43 +98,40 @@ const AGENT_SEAT_SCRIPTS: ScenarioScript[] = [ steps: [{ text: "[reply:talk-to-seat] Noted, I have your message." }], }, { - // A post to a mailbox the seat is a member of (FIX-1590). The seat hears - // it as ` in : `, so the marker is still in the turn. + // A post a coordinator delivered to the seat (FIX-1590). The seat hears it + // as `, through : `, so the marker is still in the + // turn. Its answer lands in the person's conversation under its name. match: (turn) => turn.includes("[scenario:wake]"), - steps: [{ text: "[reply:wake] Heard it in the mailbox." }], + steps: [{ text: "[reply:wake] Heard it." }], }, { - // A post the seat answers in the mailbox itself (FIX-1594). The first step - // calls `post-to-mailbox` with no text, so the real tool runs, into the - // mailbox the heard turn names; the second says so in the seat's own - // conversation. The line carries `[reply:in-mailbox]` and the post's - // `reply-token-…`, never the scenario marker, so a seat that hears the - // line does not answer it again. A seat without the tool posts nothing. + // A post the seat answers naming the post's `reply-token-…` (FIX-1594): the + // answer that lands in the person's conversation carries the token and + // `[reply:in-mailbox]`, never the scenario marker, so a seat handed the + // answer does not answer it again. match: (turn) => turn.includes("[scenario:reply-in-mailbox]"), - steps: replyInMailbox, + steps: replyWithToken, }, { // The same answer, held about three seconds first, so a page can be seen // showing the seat as working on the post before its line lands. match: (turn) => turn.includes("[scenario:reply-after-a-hold]"), holdMs: 3_000, - steps: replyInMailbox, + steps: replyWithToken, }, { - // The wake's text answer, held about three seconds first. It never calls - // the post tool, so on a routed post its line lands only as the kind's own - // landing, and the hold outlasts the person's own post request: a page is - // seen showing the seat as working before that line, and a page that does - // not follow the mailbox live never sees the line without a reload. + // The wake's answer, held about three seconds first, so the hold outlasts + // the person's own post request: a page is seen showing the seat as working + // before its line lands, and a page that does not follow the conversation + // live never sees the line without a reload. match: (turn) => turn.includes("[scenario:wake-after-a-hold]"), holdMs: 3_000, - steps: [{ text: "[reply:wake] Heard it in the mailbox." }], + steps: [{ text: "[reply:wake] Heard it." }], }, { // What the seat can see from before this turn (FIX-1611 BR-19): every - // token in the system messages (where a routed post's recent lines land) - // and in the earlier turns of this conversation, and none from the turn - // itself. The reply names nothing it could not see, so a check reads what + // token in the system messages and in the earlier turns of this + // conversation, and none from the turn itself. The reply names nothing it could not see, so a check reads what // reached the model off the reply. match: (turn) => turn.includes("[scenario:recall]"), steps: (_turn, _toolResults, messages) => { @@ -142,71 +139,21 @@ const AGENT_SEAT_SCRIPTS: ScenarioScript[] = [ return [{ text: `[reply:recall] ${seen.length > 0 ? seen.join(" ") : "Nothing came before this."}` }]; }, }, - { - // A case that needs a person (FIX-1611 BR-20). The first step calls - // `escalate` with the post's token in the case, so the real tool runs; the - // second says it filed, or, when the tool reports it filed nothing, says - // that instead. `[forge-author]` makes the call also name an author, a - // board and a mailbox of its own, which the tool's input does not carry. - match: (turn) => turn.includes("[scenario:needs-a-person]"), - steps: (turn, toolResults) => { - const token = TOKEN.exec(turn)?.[0] ?? "no-token"; - const forged = turn.includes("[forge-author]") - ? { author: "support.fsd", board: "followups", mailbox: "support.elsewhere" } - : {}; - return [ - { - toolCalls: [ - { - toolCallId: `tc_${token}`, - toolName: "escalate", - args: { case: `Needs a person: ${token}`, ...forged }, - }, - ], - }, - unfiled(toolResults) - ? { text: "[reply:unfiled] Nothing was filed: filing is unavailable here." } - : { text: "[reply:escalated] Filed onto escalations." }, - ]; - }, - }, ]; /** - * A seat's answer in the mailbox its heard turn names: `post-to-mailbox` with - * no text first, so the real tool runs, then a line in the seat's own - * conversation. The line carries `[reply:in-mailbox]` and the post's - * `reply-token-…`, never a scenario marker, so a seat that hears the line does - * not answer it again. + * A seat's answer naming the post's `reply-token-…`, so a check can tell which + * post a landed line answers. It carries `[reply:in-mailbox]`, never a scenario + * marker, so a seat handed the answer does not answer it again. */ -function replyInMailbox(turn: string): MockGeneratorScriptStep[] { - const mailbox = /^"?\S+ in ([^\s:]+): /.exec(turn)?.[1] ?? "no-mailbox"; +function replyWithToken(turn: string): MockGeneratorScriptStep[] { const token = /reply-token-[a-z0-9]+/.exec(turn)?.[0] ?? "no-token"; - return [ - { - toolCalls: [ - { - toolCallId: `tc_${token}`, - toolName: "post-to-mailbox", - args: { mailbox, body: `[reply:in-mailbox] ${token} Refunds post on Fridays.` }, - }, - ], - }, - { text: "[reply:in-mailbox] Answered in the mailbox." }, - ]; + return [{ text: `[reply:in-mailbox] ${token} Refunds post on Fridays.` }]; } /** A token a check mints into a post, and the only thing recall names. */ const TOKEN = /[a-z]+-token-[a-z0-9]+/; -/** Whether `escalate` reported that it filed nothing. */ -function unfiled(toolResults: MockToolResult[]): boolean { - return toolResults.some( - (entry) => - entry.toolName === "escalate" && (entry.result as { filed?: unknown } | null)?.filed === false, - ); -} - type Message = { role?: unknown; content?: unknown }; /** The messages of a model call, or none when the input is not a message list. */ @@ -333,17 +280,18 @@ export function holdBeforeAnswer(generator: string, input: unknown): number { } /** - * A routed mailbox's evaluation. Reads the post from the state the route - * hands it (`{ recent, post: { from, text } }`): `[route:]` picks that - * member, where `` is the id a mailbox's `members:` lists. A post - * that names none fails the call, which sends it to the mailbox's fallback. + * A coordinator's best-fit evaluation. Reads the post from the state best fit + * hands it (`{ post: { from, text } }`): `[route:]` picks that + * delegate, where `` is an id the coordinator's `delegates:` lists. A + * post that names none fails the call, which sends it to the coordinator's + * fallback. */ -export const mailboxRouteMock = mockEvaluationModel({ +export const coordinatorRouteMock = mockEvaluationModel({ answers: ({ state }) => { const text = (state as { post?: { text?: unknown } }).post?.text; const member = typeof text === "string" ? /\[route:([^\]\s]+)\]/.exec(text)?.[1] : undefined; if (member === undefined) { - throw new Error("mailbox-route (test mode): the post names no [route:], so the call fails"); + throw new Error("coordinator-route (test mode): the post names no [route:], so the call fails"); } return { member: { type: "choice", choice: member } }; }, diff --git a/apps/kitchen-sink/lib/escalate-control.ts b/apps/kitchen-sink/lib/escalate-control.ts deleted file mode 100644 index 3f852daace..0000000000 --- a/apps/kitchen-sink/lib/escalate-control.ts +++ /dev/null @@ -1,35 +0,0 @@ -/** - * The goal check's control for filing (`GOAL_CONTROL=no-filing`). - * - * Swaps `escalate` in the `agent` kind's catalog for a stand-in under the same - * name, input and description that says it filed and files nothing. The - * specialist's line still says it filed, so the goal - * `goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it` must FAIL at the - * board row alone under it: a "filed" line is never proof of a row. - * - * Honoured only under `KITCHEN_SINK_TEST_MODE=1` (`goalControl`), so a control - * can never reach a deployed build. - */ -import { handler, type BlockDefinition } from "@flow-state-dev/core"; -import { z } from "zod"; - -import { ESCALATE_DESCRIPTION, escalateInput } from "../workforce/blocks/escalate"; -import { goalControl } from "./goal-control"; - -/** - * The catalog with `escalate` swapped for the stand-in when the control is - * on; `undefined` otherwise. - */ -export function escalateControl( - catalog: Record>, -): Record> | undefined { - if (goalControl() !== "no-filing") return undefined; - const saysItFiled = handler({ - name: "escalate", - description: ESCALATE_DESCRIPTION, - inputSchema: escalateInput, - outputSchema: z.object({ filed: z.literal(true) }), - execute: () => ({ filed: true as const }), - }); - return { ...catalog, escalate: saysItFiled }; -} diff --git a/apps/kitchen-sink/lib/kitchen-sink-principal.ts b/apps/kitchen-sink/lib/kitchen-sink-principal.ts index 5135cb4cab..8112702354 100644 --- a/apps/kitchen-sink/lib/kitchen-sink-principal.ts +++ b/apps/kitchen-sink/lib/kitchen-sink-principal.ts @@ -8,8 +8,9 @@ * real organization instead, and `fsdev.config.ts` hands * {@link resolveKitchenSinkPrincipal} to `createFlowState` as the host-level * fallback. Every flow that declares no resolver of its own resolves through - * it: the assistant's flow, every seat and every mailbox. That is what makes a - * hire and the read that follows it land in the same organization. + * it: the assistant's flow and every worker flow, the coordinator among them. + * That is what makes a hire and the read that follows it land in the same + * organization. * * Two flows keep a resolver of their own. `workforce-admin` authenticates an * operator by bearer token, and every token must name @@ -32,7 +33,7 @@ import type { ResolvePrincipalFn } from "@flow-state-dev/core/types"; /** The one organization this app runs as. The only one `WORKFORCE_ADMIN_TOKENS` may name. */ export const KITCHEN_SINK_ORG_ID = "kitchen-sink"; -/** The one user every caller of this app is, including the boot that opens the mailboxes. */ +/** The one user every caller of this app is. */ export const KITCHEN_SINK_USER_ID = "devuser"; /** diff --git a/apps/kitchen-sink/lib/landing-control.ts b/apps/kitchen-sink/lib/landing-control.ts new file mode 100644 index 0000000000..d971849526 --- /dev/null +++ b/apps/kitchen-sink/lib/landing-control.ts @@ -0,0 +1,112 @@ +/** + * The goal check's control for an answer's landing (`GOAL_CONTROL=no-landing`). + * + * Swaps the built-in `agent` flow for one that takes a coordinator's delegated + * post and answers it with the worker's instructions and the tools its + * `tools:` names, in its own conversation, and hands nothing back: the answer + * stays in the specialist's conversation and never reaches the person's + * conversation with the coordinator. That is all it takes away. A check that + * the specialist's answer lands under its name must FAIL under it. + * + * Everything else a check reads stays as the real flow has it: the + * specialist's answers are drawn and kept in its own conversation, and a + * person's direct conversation with a specialist still answers. With no answer + * landing, best fit's hold stays on the specialist that took the person's + * first post. + * + * Honoured only under `KITCHEN_SINK_TEST_MODE=1` (`goalControl`), so a control + * can never reach a deployed build. The published flow has no switch that + * stops an answer landing, and must not. + */ +import { defineFlow, generator, handler, sequencer, type BlockDefinition } from "@flow-state-dev/core"; +import { + AGENT_KIND, + delegatedPostSchema, + workerConfigOf, + workerConfigSchema, + type DelegatedPost, + type WorkerInstallation, +} from "@flow-state-dev/workforce"; +import { z } from "zod"; + +import { goalControl } from "./goal-control"; + +/** A delegated post as the specialist's turn reads it, the way the real flow words it. */ +const heard = (post: DelegatedPost) => `${post.from}, through ${post.coordinator}: ${post.body}`; + +/** + * The quiet `agent` flow when the control is on; `undefined` otherwise. + * + * @param catalog The tool catalog the real flow carries. A worker reaches the + * entries its `tools:` names, as on the real flow. + * @param installation The installation the real flow runs its workers on: + * this one does too, loading each turn's worker first, as the real flow does. + */ +export function landingControl(catalog: Record>, installation: WorkerInstallation) { + if (goalControl() !== "no-landing") return undefined; + const settings = workerConfigSchema().extend({ tools: z.array(z.string()).optional() }); + type Settings = z.infer; + const config = (ctx: { session: object }) => workerConfigOf(ctx) as Settings; + const tools = (_input: unknown, ctx: { session: object }): BlockDefinition[] => + (config(ctx).tools ?? []).filter((name) => catalog[name] !== undefined).map((name) => catalog[name]!); + const prompt = [ + (_input: unknown, ctx: { session: object }) => config(ctx).teamInstructions, + (_input: unknown, ctx: { session: object }) => config(ctx).instructions, + ]; + const answerPost = generator({ + name: "agent-answer", + inputSchema: delegatedPostSchema, + model: "intent/chat", + itemVisibility: { client: true, history: true }, + history: true, + prompt, + user: heard, + tools, + }); + const answerTurn = generator({ + name: "agent-answer", + inputSchema: z.object({ message: z.string() }), + model: "intent/chat", + itemVisibility: { client: true, history: true }, + history: true, + prompt, + user: (input: { message: string }) => input.message, + tools, + }); + // The turn's worker, loaded before the answer reads its settings. + const resolve = (inputSchema: T) => + handler({ + name: "agent-resolve-worker", + inputSchema, + resources: { ...installation.resources }, + execute: async (_input, ctx) => { + await installation.resolveWorker(ctx, AGENT_KIND); + }, + }); + const turnInput = z.object({ message: z.string() }); + return defineFlow({ + kind: AGENT_KIND, + cardinality: "collection", + configSchema: settings, + session: installation.session(), + resources: { ...installation.resources }, + actions: { + run: { + inputSchema: turnInput.strict(), + block: sequencer({ name: "agent-run", inputSchema: turnInput }).tap(resolve(turnInput)).step(answerTurn), + userMessage: (input: { message: string }) => input.message, + }, + }, + internal: { + actions: { + onDelegatedPost: { + inputSchema: delegatedPostSchema, + block: sequencer({ name: "agent-heard-post", inputSchema: delegatedPostSchema }) + .tap(resolve(delegatedPostSchema)) + .step(answerPost), + userMessage: heard, + }, + }, + }, + } as never); +} diff --git a/apps/kitchen-sink/lib/mailbox-landing-control.ts b/apps/kitchen-sink/lib/mailbox-landing-control.ts deleted file mode 100644 index 55c00f8e19..0000000000 --- a/apps/kitchen-sink/lib/mailbox-landing-control.ts +++ /dev/null @@ -1,146 +0,0 @@ -/** - * The goal check's control for a routed answer's landing (`GOAL_CONTROL=no-landing`). - * - * Swaps the built-in `agent` kind for one that hears posts and answers them - * with the seat's instructions, the mailbox's recent lines and the tools its - * `tools:` names, and lands no text answer: a text answer stays in the seat's - * own conversation and never reaches the mailbox. That is all it takes away. The - * goal `goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it` must FAIL - * its answer leg under it, and the line that says a case was filed, while the - * filed row still lands. - * - * Everything else a check reads stays as the real kind has it. The seat's - * answers are drawn and kept in its own conversation. `post-to-mailbox` is - * carried, from the same capability: a line the seat posts with the tool is - * not a landing, and still reaches the mailbox under the seat's name, so the - * route stops holding the person's next post for that seat. Nothing here marks - * a turn as routed, so the tool posts the line as the seat's own post, not as - * the answer the mailbox keeps for the post. - * - * The same stand-in FIX-1610's own check uses, with the kind's tools added so - * `escalate` still files and `post-to-mailbox` still posts. - * - * Honoured only under `KITCHEN_SINK_TEST_MODE=1` (`goalControl`), so a control - * can never reach a deployed build. The published kind has no switch that - * stops a routed answer landing, and must not. - */ -import { defineFlow, generator, handler, sequencer, type BlockDefinition, type DefinedCapability } from "@flow-state-dev/core"; -import { resolveActivePresets } from "@flow-state-dev/core/capability"; -import { - AGENT_KIND, - mailboxNotifyInputSchema, - workerConfigOf, - workerConfigSchema, - type MailboxNotifyInput, - type WorkerInstallation, -} from "@flow-state-dev/workforce"; -import { z } from "zod"; - -import { goalControl } from "./goal-control"; - -/** A seat's turn as a post is heard: ` in : `. */ -const heard = (post: MailboxNotifyInput) => `${post.author ?? post.principal} in ${post.mailboxId}: ${post.body}`; - -/** - * The tools a capability's active presets grant, by name: what the real kind - * adds to its catalog from its `uses`, so a seat's `tools:` can name them. - */ -function grantedTools(capability: DefinedCapability): Record> { - return Object.fromEntries( - resolveActivePresets(capability) - .flatMap(({ preset }) => (Array.isArray(preset.tools) ? preset.tools : [])) - .map((tool) => [tool.name, tool]), - ); -} - -/** - * The quiet `agent` kind when the control is on; `undefined` otherwise. - * - * @param catalog The tool catalog the real kind carries. - * @param mailboxPost The mailbox-post capability the real kind is built with. Its - * tool joins the catalog, and a seat reaches the entries its `tools:` names, - * as on the real kind. - * @param installation The installation the real kind runs its workers on: - * this one does too, loading each turn's worker first, as the real kind does. - */ -export function mailboxLandingControl( - catalog: Record>, - mailboxPost: DefinedCapability, - installation: WorkerInstallation, -) { - if (goalControl() !== "no-landing") return undefined; - const kindCatalog = { ...grantedTools(mailboxPost), ...catalog }; - const settings = workerConfigSchema().extend({ tools: z.array(z.string()).optional() }); - type Settings = z.infer; - const config = (ctx: { session: object }) => workerConfigOf(ctx) as Settings; - const tools = (_input: unknown, ctx: { session: object }): BlockDefinition[] => - (config(ctx).tools ?? []) - .filter((name) => kindCatalog[name] !== undefined) - .map((name) => kindCatalog[name]!); - const prompt = [ - (_input: unknown, ctx: { session: object }) => config(ctx).teamInstructions, - (_input: unknown, ctx: { session: object }) => config(ctx).instructions, - ]; - const answerPost = generator({ - name: "agent-answer", - inputSchema: mailboxNotifyInputSchema, - model: "intent/chat", - itemVisibility: { client: true, history: true }, - history: true, - prompt, - context: [ - (post: MailboxNotifyInput) => - post.recent === undefined || post.recent.length === 0 - ? undefined - : ["Recent lines in the mailbox, oldest first:", ...post.recent.map((l) => `- ${l.author ?? l.principal}: ${l.body}`)].join("\n"), - ], - user: heard, - tools, - }); - const answerTurn = generator({ - name: "agent-answer", - inputSchema: z.object({ message: z.string() }), - model: "intent/chat", - itemVisibility: { client: true, history: true }, - history: true, - prompt, - user: (input: { message: string }) => input.message, - tools, - }); - // The turn's worker, loaded before the answer reads its settings. - const resolve = (inputSchema: T) => - handler({ - name: "agent-resolve-worker", - inputSchema, - resources: { ...installation.resources }, - execute: async (_input, ctx) => { - await installation.resolveWorker(ctx, AGENT_KIND); - }, - }); - const turnInput = z.object({ message: z.string() }); - return defineFlow({ - kind: AGENT_KIND, - cardinality: "collection", - configSchema: settings, - session: installation.session(), - resources: { ...installation.resources }, - actions: { - run: { - inputSchema: turnInput.strict(), - block: sequencer({ name: "agent-run", inputSchema: turnInput }).tap(resolve(turnInput)).step(answerTurn), - userMessage: (input: { message: string }) => input.message, - }, - }, - internal: { - actions: { - onMailboxPost: { - inputSchema: mailboxNotifyInputSchema, - block: sequencer({ name: "agent-heard-post", inputSchema: mailboxNotifyInputSchema }) - .tap(resolve(mailboxNotifyInputSchema)) - .step(answerPost), - userMessage: heard, - }, - }, - }, - } as never); -} diff --git a/apps/kitchen-sink/lib/mailbox-post-control.ts b/apps/kitchen-sink/lib/mailbox-post-control.ts deleted file mode 100644 index 6bb3b9d6bf..0000000000 --- a/apps/kitchen-sink/lib/mailbox-post-control.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * The goal check's control for a seat's post (`GOAL_CONTROL=post-without-author`). - * - * Swaps the published mailbox-post capability for a stand-in whose - * `post-to-mailbox` dispatches the same `post` into the same mailbox with no - * `author`: the line then reads as the request's principal (`devuser`), not as - * the seat. The goal `goals/kitchen-sink-talk/agent-replies-in-the-mailbox` - * must FAIL its "under its own name" leg under it. - * - * The published tool has no switch that drops the author, and must not: the - * name on the line is the seat's `seatId`. Dropping it is what this control - * grades. Wakes stay withheld either way, because the dispatch is the seat - * post action. The stand-in lives here, honoured only under `KITCHEN_SINK_TEST_MODE=1` - * (`goalControl`), and can never reach a deployed build. - */ -import { defineCapability, dispatcher, type DefinedCapability } from "@flow-state-dev/core"; -import { - MAILBOX_KIND, - MAILBOX_SEAT_POST_ACTION, - MAILBOX_POST_CAPABILITY, - POST_TO_MAILBOX_TOOL, - postToMailboxInputSchema, - type PostToMailboxInput, -} from "@flow-state-dev/workforce"; - -import { goalControl } from "./goal-control"; - -/** The stand-in capability when the control is on; `undefined` otherwise. */ -export function mailboxPostControl(): DefinedCapability | undefined { - if (goalControl() !== "post-without-author") return undefined; - const postWithoutAuthor = dispatcher({ - name: POST_TO_MAILBOX_TOOL, - description: "Post a line to a mailbox you are a member of.", - flowKind: MAILBOX_KIND, - action: MAILBOX_SEAT_POST_ACTION, - inputSchema: postToMailboxInputSchema, - session: { id: (input: PostToMailboxInput) => input.mailbox }, - payload: (input: PostToMailboxInput) => ({ body: input.body }), - }); - return defineCapability({ - name: MAILBOX_POST_CAPABILITY, - presets: { tools: { tools: [postWithoutAuthor] }, default: ["tools"] }, - }); -} diff --git a/apps/kitchen-sink/lib/mailbox-route-control.ts b/apps/kitchen-sink/lib/mailbox-route-control.ts deleted file mode 100644 index a8aed4606a..0000000000 --- a/apps/kitchen-sink/lib/mailbox-route-control.ts +++ /dev/null @@ -1,20 +0,0 @@ -/** - * The goal checks' control for a mailbox's route (`GOAL_CONTROL=no-route`). - * - * Takes the `routing:` lines off every mailbox as it is read, before the - * mailboxes are bound, so `support.help` is a plain mailbox again: every - * member hears every post, and nothing lands on its own. A check that one - * specialist answers and nobody else runs must FAIL under it. - * - * Honoured only under `KITCHEN_SINK_TEST_MODE=1` (`goalControl`), so a control - * can never reach a deployed build. - */ -import type { MailboxManifest } from "@flow-state-dev/workforce"; - -import { goalControl } from "./goal-control"; - -/** The mailboxes as read, or with their `routing:` removed when the control is on. */ -export function withMailboxRouteControl(mailboxes: MailboxManifest[]): MailboxManifest[] { - if (goalControl() !== "no-route") return mailboxes; - return mailboxes.map(({ declared: { routing: _routing, ...declared }, ...rest }) => ({ ...rest, declared })); -} diff --git a/apps/kitchen-sink/lib/mailbox-wake-control.ts b/apps/kitchen-sink/lib/mailbox-wake-control.ts deleted file mode 100644 index 0f0740ea20..0000000000 --- a/apps/kitchen-sink/lib/mailbox-wake-control.ts +++ /dev/null @@ -1,60 +0,0 @@ -/** - * The goal checks' controls for the mailbox wake (`GOAL_CONTROL`). - * - * - `name-only-notify` puts back the notify block every member got before the - * wake existed: a name-only line, and no seat runs. The goal - * `goals/kitchen-sink-talk/a-post-runs-each-member-agent-once` must FAIL - * under it. - * - `no-author-filter` hands the wake each delivery with its `seatAuthored` - * mark dropped, so a post a seat wrote wakes the agent members too. A check - * that a seat's own post runs no seat must FAIL under it. The name is the - * control the checks already set; the mark they drop is the one the wake - * reads. - * - * Both wrap Workforce's `wakeMemberSeats` from outside: the package has no - * switch for either, and a test control never reaches it. - * - * Honoured only under `KITCHEN_SINK_TEST_MODE=1` (`goalControl`), so a control - * can never reach a deployed build. A test seam, kept in `lib/` so the notify - * block does not import from `test/`. - */ -import { handler, type BlockDefinition } from "@flow-state-dev/core"; -import { mailboxNotifyInputSchema, type MailboxNotifyInput } from "@flow-state-dev/workforce"; -import { z } from "zod"; - -import { goalControl } from "./goal-control"; - -/** The mailbox-wake control in force, or `undefined` when none is. */ -function mailboxWakeControl(): "name-only-notify" | "no-author-filter" | undefined { - const control = goalControl(); - return control === "name-only-notify" || control === "no-author-filter" ? control : undefined; -} - -/** - * `name-only-notify`'s stand-in: a transient line naming the member, and no - * seat runs. Nobody is told about their own post. - */ -const nameOnly = handler({ - name: "kitchen-sink-notify-member", - inputSchema: mailboxNotifyInputSchema, - outputSchema: z.object({ notified: z.string() }), - execute: (input: MailboxNotifyInput, ctx) => { - if (input.author !== undefined && input.member === input.author) return { notified: "" }; - ctx.emit.message(`[${input.mailboxId}] → ${input.member}: ${input.body}`, { transient: true }); - return { notified: input.member }; - }, -}); - -/** - * The notify block with the control in force applied: `wake` unchanged when - * none is, the name-only stand-in in its place under `name-only-notify`, and - * `wake` fed deliveries with the seat mark dropped under `no-author-filter`. - */ -export function withMailboxWakeControl(wake: BlockDefinition): BlockDefinition { - const control = mailboxWakeControl(); - if (control === "name-only-notify") return nameOnly; - if (control === "no-author-filter") { - return wake.connectInput(({ seatAuthored: _seatAuthored, ...post }: MailboxNotifyInput) => post); - } - return wake; -} diff --git a/apps/kitchen-sink/lib/models.ts b/apps/kitchen-sink/lib/models.ts index 4a5bb11341..f579bdb209 100644 --- a/apps/kitchen-sink/lib/models.ts +++ b/apps/kitchen-sink/lib/models.ts @@ -25,13 +25,13 @@ export const KITCHEN_SINK_MODELS = [ ] as const; /** - * The model `support.help`'s route asks which specialist answers a post. + * The model `support.help`'s best fit asks which specialist answers a post. * - * An evaluation model, not a chat model: the route's one call is an + * An evaluation model, not a chat model: best fit's one call is an * evaluator, and a chat model named here is refused when the call resolves. * Being an evaluation model it is also untouched by the dev override that * forces every generator onto the cheap model. Test mode scripts the call by - * its block name, `mailbox-route`, and never reaches this model. + * its block name, `coordinator-route`, and never reaches this model. */ export const ROUTE_MODEL = "vercel/typesafe-ai/jev"; diff --git a/apps/kitchen-sink/lib/rail-sessions.ts b/apps/kitchen-sink/lib/rail-sessions.ts index 1b854a8f46..b15fa98694 100644 --- a/apps/kitchen-sink/lib/rail-sessions.ts +++ b/apps/kitchen-sink/lib/rail-sessions.ts @@ -1,33 +1,43 @@ /** - * The rail's session reads: a mailbox kind's sessions are the mailboxes the - * tree declares, read by id; every other kind's are listed as the store has - * them. + * The rail's session reads: the coordinator kind's sessions are the person's + * conversations with the coordinators the tree declares, one each, found or + * started by the coordinator's id; every other kind's are listed as the store + * has them. * - * The navigator lists a singleton kind's sessions by kind. A store kept across - * an upgrade still holds the sessions of mailboxes an earlier roster declared - * (the old desk's `support.desk`, say), and a listing by kind would draw them - * beside `support.help`. Reading each declared mailbox by its id never reads a - * retired one (BP-033), and nothing stored is changed or removed. + * A coordinator is a worker, so a person's conversation with `support.help` is + * a session on the `coordinator` flow that names it, found the way any + * worker's is (`ensureWorkerSession({ worker })`, which matches on the + * criteria's keys and so never returns a delegate's session). Each row reads + * as the coordinator's id, the name the person knows it by. */ -import type { SessionClient } from "@flow-state-dev/client"; +import type { SessionClient, SessionSummary } from "@flow-state-dev/client"; +import type { WorkforceClient } from "@flow-state-dev/workforce/browser"; -import { isMailboxKind, SHELL_MAILBOXES } from "./workforce-shell"; +import { isCoordinatorKind, SHELL_COORDINATORS } from "./workforce-shell"; /** * The session source the rail's navigator reads through. * * @param client The page's session client; a stable reference, so the navigator's reads stay fenced on it. - * @returns A source whose mailbox-kind listings are the declared mailboxes. + * @param workforce The person's workforce client, which finds or starts their conversation with a coordinator. + * @returns A source whose coordinator-kind listing is one conversation per declared coordinator. */ export function railSessions( - client: Pick, + client: Pick, + workforce: Pick, ): Pick { return { listSessions: async (options) => { const kind = options?.flowKind; - if (kind === undefined || !isMailboxKind(kind)) return await client.listSessions(options); - const declared = SHELL_MAILBOXES.filter((mailbox) => mailbox.kind === kind); - return await Promise.all(declared.map((mailbox) => client.getSession(mailbox.id))); + if (kind === undefined || !isCoordinatorKind(kind)) return await client.listSessions(options); + return await Promise.all( + SHELL_COORDINATORS.map( + async (worker): Promise => ({ + ...(await workforce.ensureWorkerSession({ worker })), + title: worker, + }), + ), + ); }, }; } diff --git a/apps/kitchen-sink/lib/workforce-shell.ts b/apps/kitchen-sink/lib/workforce-shell.ts index a5db6a2e9a..b554443cb0 100644 --- a/apps/kitchen-sink/lib/workforce-shell.ts +++ b/apps/kitchen-sink/lib/workforce-shell.ts @@ -1,6 +1,6 @@ /** * The names the shell and its flow agree on — which kinds the rail groups, - * what a worker's composer sends, and which boards the right panel reads. + * which coordinators it lists, and what a worker's composer sends. * * A leaf module on purpose (BP-019): the page is a client bundle and the flow * is server code, and both import these. Nothing here imports anything, so the @@ -10,39 +10,42 @@ * roster through Workforce's client (`@flow-state-dev/workforce/browser`), * which answers with each worker's flow and description. * - * The kind names and board ids are written down rather than derived, because - * the only place that knows them at run time is the server-side tree under - * `workforce/`, which a browser cannot read. `test/workforce-shell.test.ts` - * holds them to that tree and fails when a kind or a board changes there and - * not here. + * The kind names and coordinator ids are written down rather than derived, + * because the only place that knows them at run time is the server-side tree + * under `workforce/`, which a browser cannot read. `test/workforce-shell.test.ts` + * holds them to that tree and fails when a kind or a coordinator changes there + * and not here. */ /** The flow the stream talks to. */ export const SHELL_FLOW_KIND = "chat-agent"; /** - * The mailbox kinds: the framework's own `mailbox`, plus every kind under - * `workforce/flows/mailboxes/`, of which this app has none. + * The coordinator flow's kind: a worker whose `flow:` names it hands each post + * to its delegates. Its panel shows the conversation's lines, each under who + * wrote it, rather than a conversation with one worker. */ -export const MAILBOX_KINDS = ["mailbox"] as const; +export const COORDINATOR_KINDS = ["coordinator"] as const; -/** Whether a picked flow kind is a mailbox kind, whose panel shows a transcript rather than a conversation. */ -export function isMailboxKind(kind: string): boolean { - return (MAILBOX_KINDS as readonly string[]).includes(kind); +/** Whether a picked flow kind is a coordinator's, whose panel shows lines under their writers' names. */ +export function isCoordinatorKind(kind: string): boolean { + return (COORDINATOR_KINDS as readonly string[]).includes(kind); } /** - * The mailboxes the tree declares, each with the kind its `MAILBOX.md` selects: - * what the rail lists under that kind. A store kept across an upgrade can still - * hold the sessions of mailboxes the tree no longer declares, so the rail reads - * these by id rather than listing the kind (`lib/rail-sessions.ts`). + * The coordinators the tree declares, by worker id: what the rail lists under + * the coordinator kind. Each is the person's own conversation with that + * coordinator, found or started by its id (`lib/rail-sessions.ts`). */ -export const SHELL_MAILBOXES = [{ id: "support.help", kind: "mailbox" }] as const; +export const SHELL_COORDINATORS = ["support.help"] as const; + +/** What a coordinator's composer sends: the person's post, through the flow's door. */ +export const COORDINATOR_ASK = { action: "run", field: "message" } as const; /** - * The worker flows: the built-in `agent`, plus every flow under - * `workforce/flows/workers/`, of which this app has none. Each runs as one - * copy, holding every worker's conversations on it. + * The worker flows a person talks to one worker on: the built-in `agent`, plus + * every flow under `workforce/flows/workers/`, of which this app has none. Each + * runs as one copy, holding every worker's conversations on it. */ export const SEAT_KINDS = ["agent"] as const; @@ -66,14 +69,21 @@ export const SEAT_ASKS = { } as const satisfies Record<(typeof SEAT_KINDS)[number], SeatAsk>; /** - * The worker a mailbox woke, read off the run's `topic`: the key Workforce's - * wake derives each worker's conversation from, `mailbox::`. - * Every worker on a flow shares its copy, so the run's `flowId` names the - * flow (`agent`), not the worker. `undefined` for any other run. + * The delegate a coordinator handed a post to, read off the run's `topic`: the + * key Workforce's coordinator derives each delegate's session from, + * `delegate:<[worker, target]>`. Every worker on a flow shares its copy, so + * the run's `flowId` names the flow (`agent`), not the worker. `undefined` for + * any other run. */ -export function wokenWorkerOf(run: { readonly topic?: string }): string | undefined { - const match = /^mailbox:[^:]+:(.+)$/.exec(run.topic ?? ""); - return match?.[1]; +export function delegateOfRun(run: { readonly topic?: string }): string | undefined { + const match = /^delegate:(.+)$/.exec(run.topic ?? ""); + if (match === null) return undefined; + try { + const key = JSON.parse(match[1]!) as unknown; + return Array.isArray(key) && typeof key[0] === "string" ? key[0] : undefined; + } catch { + return undefined; + } } /** A seat kind's answer, or `undefined` for a kind the shell does not know. */ @@ -101,15 +111,3 @@ export function defaultConversation !(session.tags?.includes(LEGACY_SEAT_HIRES_TAG) ?? false)); } - -/** - * The boards the right panel draws, one per `boards:` entry in the tree's - * `MAILBOX.md` files. - * - * `ref` is the ledger id the workforce package mints for that pair. The panel - * reads each board through its mailbox's session, whose flow declares it under - * that id, and the test checks the two agree. - */ -export const SHELL_BOARDS = [ - { mailboxId: "support.help", board: "escalations", ref: "support.help.escalations" }, -] as const; diff --git a/apps/kitchen-sink/test/client-imports.test.ts b/apps/kitchen-sink/test/client-imports.test.ts index cf40631a83..84ceb1ae16 100644 --- a/apps/kitchen-sink/test/client-imports.test.ts +++ b/apps/kitchen-sink/test/client-imports.test.ts @@ -2,10 +2,10 @@ * A client component takes workforce names from `@flow-state-dev/workforce/browser`, * never from the package root. * - * The root carries the mailbox floor, which reaches the orchestration task + * The root carries the server-side flows, which reach the orchestration task * board and `node:async_hooks`. A dev bundler that keeps a module a client * component names fails the page on that import: `GET /` answered 500 when the - * picked-mailbox panel took `MAILBOX_POST_COMPONENT` from the root. The + * picked session's panel took a component name from the root. The * subpath's own import graph is held Node-free by * `packages/workforce/test/browser-subpath-safe.test.ts`; this holds the app to * using it. diff --git a/apps/kitchen-sink/test/e2e-mock-script.test.ts b/apps/kitchen-sink/test/e2e-mock-script.test.ts index 0ff9183c19..72cf34e8f0 100644 --- a/apps/kitchen-sink/test/e2e-mock-script.test.ts +++ b/apps/kitchen-sink/test/e2e-mock-script.test.ts @@ -2,36 +2,29 @@ * The scripted model's scenario dispatcher, which every keyless check in this * app runs on. * - * Two promises, and why each matters: + * A scenario is picked by a marker in the LATEST user turn only. A seat's + * conversation carries its earlier turns, so matching the whole history would + * answer a new message with an old message's scenario. Red state, before the + * green was trusted: the dispatcher before FIX-1589 (whole-history match) + * failed the case below. * - * - A scenario is picked by a marker in the LATEST user turn only. A seat's - * conversation carries its earlier turns, so matching the whole history - * would answer a new message with an old message's scenario. - * - Each request walks its own copy of a scenario's steps. A multi-step - * scenario (a tool call, then the reply) run twice in one process must - * start at its first step both times; one shared cursor made the second - * run start mid-script and skip the tool call. - * - * Red state, before the green was trusted: the dispatcher before FIX-1589 - * (whole-history match, one cursor per scenario) failed both cases. - * - * And the two scenarios FIX-1611 adds (`specs/issues/FIX-1611/BUSINESS-RULES.md`, + * And the scenarios FIX-1611 adds (`specs/issues/FIX-1611/BUSINESS-RULES.md`, * V3): BR-19 `[scenario:recall]` names every token found before the latest * turn, and no other, and under `GOAL_CONTROL=no-history` only those in the - * system message (BR-23); BR-20 `[scenario:needs-a-person]` calls `escalate` - * once with the post's token, then says it filed, or that it did not; BR-21 - * `[route:]` routes, below. + * system message (BR-23); BR-21 `[route:]` routes, below. + * `[scenario:needs-a-person]` went with kitchen-sink's escalation feature + * (FIX-1792). * * A held scenario (`[scenario:reply-after-a-hold]`) gives a page time to show * a seat as working before its line lands. Its line must wait out the hold, * and nothing else may hold: every other check's timing depends on that. - * `[scenario:wake-after-a-hold]` holds the same way before a text answer, which - * lands by the kind's own landing rather than the post tool. Without the hold - * that answer lands before a page can show the seat working, and before the - * person's own post request ends and the page reads the mailbox again anyway. + * `[scenario:wake-after-a-hold]` holds the same way before the wake's answer. + * Without the hold that answer lands before a page can show the seat working, + * and before the person's own post request ends and the page reads the + * conversation again anyway. */ import { afterEach, describe, expect, it, vi } from "vitest"; -import { agentSeatMock, mailboxRouteMock, holdBeforeAnswer } from "@/lib/e2e-mock-script"; +import { agentSeatMock, coordinatorRouteMock, holdBeforeAnswer } from "@/lib/e2e-mock-script"; import { createKitchenSinkTestModelResolver } from "./mock-flowstate"; const system = (content: string) => ({ role: "system", content }); @@ -43,31 +36,16 @@ afterEach(() => { }); describe("the scripted model's scenario dispatcher", () => { - it("starts a multi-step scenario at its first step on every request", () => { - agentSeatMock.reset(); - const turn = "[scenario:needs-a-person] case-token-aaa the charger caught fire"; - - const first = [user(turn)]; - expect(agentSeatMock.next(first)?.toolCalls?.[0]?.toolName).toBe("escalate"); - expect(agentSeatMock.next(first)?.text).toContain("[reply:escalated]"); - - // A second request with the very same turn: a fresh messages array, so a - // fresh walk of the script. - const second = [user(turn)]; - expect(agentSeatMock.next(second)?.toolCalls?.[0]?.toolName).toBe("escalate"); - expect(agentSeatMock.next(second)?.text).toContain("[reply:escalated]"); - }); - it("matches the latest user turn only, not an earlier turn in the conversation", () => { agentSeatMock.reset(); const history = [ - user("[scenario:needs-a-person] case-token-bbb please escalate"), - assistant("[reply:escalated] Filed onto escalations."), + user("devuser, through support.help: [scenario:reply-in-mailbox] reply-token-bbb when do refunds post?"), + assistant("[reply:in-mailbox] reply-token-bbb Refunds post on Fridays."), user("[scenario:talk-to-seat] where is my refund?"), ]; const step = agentSeatMock.next(history); - expect(step?.toolCalls).toBeUndefined(); expect(step?.text).toContain("[reply:talk-to-seat]"); + expect(step?.text).not.toContain("reply-token-bbb"); // And a latest turn with no marker is unmatched, however marked the history. const unmarked = [...history, assistant("..."), user("just saying hello")]; @@ -75,47 +53,12 @@ describe("the scripted model's scenario dispatcher", () => { }); }); -describe("the needs-a-person scenario (BR-20)", () => { - it("calls escalate once with the post's token as the case, then says it filed", () => { - agentSeatMock.reset(); - const messages = [user("devuser in support.help: [route:support.devices] [scenario:needs-a-person] case-token-ccc it smokes")]; - const call = agentSeatMock.next(messages); - expect(call?.toolCalls).toHaveLength(1); - expect(call?.toolCalls?.[0]).toMatchObject({ toolName: "escalate", args: { case: expect.stringContaining("case-token-ccc") } }); - // The case alone: the script names no author, board or mailbox. - expect(Object.keys(call?.toolCalls?.[0]?.args ?? {})).toEqual(["case"]); - expect(agentSeatMock.next(messages)?.text).toMatch(/^\[reply:escalated\] /); - }); - - it("says nothing was filed when the tool reports it filed nothing", () => { - agentSeatMock.reset(); - const messages = [user("[scenario:needs-a-person] case-token-ddd it smokes")]; - agentSeatMock.next(messages); - const reply = agentSeatMock.next(messages, { - toolResults: [{ toolCallId: "tc", toolName: "escalate", result: { filed: false, reason: "unavailable" } }], - } as never); - expect(reply?.text).toMatch(/^\[reply:unfiled\] /); - expect(reply?.text).not.toContain("[reply:escalated]"); - }); - - it("with [forge-author], also names an author, a board and a mailbox of its own", () => { - agentSeatMock.reset(); - const call = agentSeatMock.next([user("[scenario:needs-a-person] [forge-author] case-token-eee refund me")]); - expect(call?.toolCalls?.[0]?.args).toEqual({ - case: expect.stringContaining("case-token-eee"), - author: "support.fsd", - board: "followups", - mailbox: "support.elsewhere", - }); - }); -}); - describe("the recall scenario (BR-19)", () => { const conversation = [ - system("You answer questions.\n\nRecent lines in the mailbox, oldest first:\n- support.devices: [reply:in-mailbox] reply-token-aaa Refunds post on Fridays."), - user("devuser in support.help: a question naming case-token-bbb"), - assistant("[reply:wake] Heard it in the mailbox."), - user("devuser in support.help: [scenario:recall] what have we said? recall-token-ccc"), + system("You answer questions.\n\nSomething from before: reply-token-aaa"), + user("devuser, through support.help: a question naming case-token-bbb"), + assistant("[reply:wake] Heard it."), + user("devuser, through support.help: [scenario:recall] what have we said? recall-token-ccc"), ]; it("names every token found before the latest turn, in the system message or an earlier turn, and no other", () => { @@ -145,10 +88,10 @@ describe("the recall scenario (BR-19)", () => { }); }); -describe("a seat that holds before answering", () => { - // How a seat hears a post: ` in : `. +describe("a seat that answers naming the post's token", () => { + // How a delegate hears a post: `, through : `. const heard = (marker: string) => [ - user(`visitor-1 in support.help: ${marker} reply-token-abc123 when do refunds post?`), + user(`devuser, through support.help: ${marker} reply-token-abc123 when do refunds post?`), ]; const held = () => heard("[scenario:reply-after-a-hold]"); const plain = () => heard("[scenario:reply-in-mailbox]"); @@ -157,6 +100,13 @@ describe("a seat that holds before answering", () => { vi.useRealTimers(); }); + it("answers in text carrying the post's token and its marker, never the scenario's", () => { + agentSeatMock.reset(); + const step = agentSeatMock.next(plain()); + expect(step?.toolCalls).toBeUndefined(); + expect(step?.text).toBe("[reply:in-mailbox] reply-token-abc123 Refunds post on Fridays."); + }); + it("holds the agent seat about three seconds for the held scenario, and nothing else", () => { expect(holdBeforeAnswer("agent-answer", held())).toBe(3_000); expect(holdBeforeAnswer("agent-answer-with-activate-tool", held())).toBe(3_000); @@ -167,59 +117,39 @@ describe("a seat that holds before answering", () => { it("answers exactly as reply-in-mailbox does once the hold ends", () => { agentSeatMock.reset(); - const heldTurn = held(); - const plainTurn = plain(); - const heldSteps = [agentSeatMock.next(heldTurn), agentSeatMock.next(heldTurn)]; - const plainSteps = [agentSeatMock.next(plainTurn), agentSeatMock.next(plainTurn)]; - expect(heldSteps).toEqual(plainSteps); - expect(heldSteps[0]?.toolCalls?.[0]).toMatchObject({ - toolName: "post-to-mailbox", - args: { mailbox: "support.help" }, - }); + expect(agentSeatMock.next(held())).toEqual(agentSeatMock.next(plain())); }); - it("keeps the seat's line back until the hold ends", async () => { + it("keeps the seat's answer back until the hold ends", async () => { vi.useFakeTimers(); agentSeatMock.reset(); - const posted: unknown[] = []; const model = createKitchenSinkTestModelResolver()("test-model", "agent-answer"); - const answer = model.generate({ - messages: held(), - tools: [ - { - name: "post-to-mailbox", - execute: async (args) => { - posted.push(args); - return { ok: true }; - }, - }, - ], - } as Parameters[0]); + let answered: string | undefined; + const answer = model + .generate({ messages: held() } as Parameters[0]) + .then((result) => (answered = result.text)); await vi.advanceTimersByTimeAsync(2_900); - expect(posted).toEqual([]); + expect(answered).toBeUndefined(); await vi.advanceTimersByTimeAsync(200); - const result = await answer; - expect(posted).toHaveLength(1); - expect(result.text).toContain("[reply:in-mailbox]"); + await answer; + expect(answered).toContain("[reply:in-mailbox] reply-token-abc123"); }); it("does not hold a post that names no hold", async () => { vi.useFakeTimers(); agentSeatMock.reset(); - const posted: unknown[] = []; const model = createKitchenSinkTestModelResolver()("test-model", "agent-answer"); - await model.generate({ - messages: plain(), - tools: [{ name: "post-to-mailbox", execute: async (args) => void posted.push(args) }], - } as Parameters[0]); - expect(posted).toHaveLength(1); + let answered: string | undefined; + void model.generate({ messages: plain() } as Parameters[0]).then((result) => (answered = result.text)); + await vi.advanceTimersByTimeAsync(0); + expect(answered).toContain("[reply:in-mailbox]"); }); }); -describe("a seat that holds, then answers in text", () => { - const heard = (marker: string) => [user(`devuser in support.help: ${marker} ask-token-abc123 is anyone there?`)]; +describe("a seat that holds, then answers the wake", () => { + const heard = (marker: string) => [user(`devuser, through support.help: ${marker} ask-token-abc123 is anyone there?`)]; const held = () => heard("[scenario:wake-after-a-hold]"); afterEach(() => { @@ -232,14 +162,14 @@ describe("a seat that holds, then answers in text", () => { expect(holdBeforeAnswer("agent-answer", heard("[scenario:wake]"))).toBe(0); }); - it("answers as the wake does, in text, and never calls the post tool", () => { + it("answers as the wake does", () => { agentSeatMock.reset(); const step = agentSeatMock.next(held()); expect(step?.toolCalls).toBeUndefined(); expect(step).toEqual(agentSeatMock.next(heard("[scenario:wake]"))); }); - it("keeps the text answer back until the hold ends", async () => { + it("keeps the answer back until the hold ends", async () => { vi.useFakeTimers(); agentSeatMock.reset(); const model = createKitchenSinkTestModelResolver()("test-model", "agent-answer"); @@ -258,45 +188,41 @@ describe("a seat that holds, then answers in text", () => { }); /** - * A routed mailbox's one evaluation, scripted. A post picks its member with - * `[route:]`; a post that names none fails the call, which is what - * sends it to the mailbox's fallback. The script reads the post from the state - * the route hands it, `{ recent, post: { from, text } }`. + * A best-fit coordinator's one evaluation, scripted. A post picks its delegate + * with `[route:]`; a post that names none fails the call, which is + * what sends it to the coordinator's fallback. The script reads the post from + * the state best fit hands it, `{ post: { from, text } }`. */ -describe("the scripted mailbox route", () => { +describe("the scripted best-fit evaluation", () => { const route = async (text: string) => - (await mailboxRouteMock.doEvaluate({ - state: { recent: [], post: { from: "devuser", text } }, + (await coordinatorRouteMock.doEvaluate({ + state: { post: { from: "devuser", text } }, questions: {} } as never)) as { answers: Record }; - it("picks the member a post names with [route:]", async () => { + it("picks the delegate a post names with [route:]", async () => { const result = await route("[route:support.devices] my laptop won't join the wifi"); expect(result.answers).toEqual({ member: { type: "choice", choice: "support.devices" } }); }); - it("fails the call for a post that names no member, so the mailbox's fallback takes it", async () => { - await expect(route("who do I ask about a parking pass?")).rejects.toThrow(/\[route:\]/); + it("fails the call for a post that names no delegate, so the coordinator's fallback takes it", async () => { + await expect(route("who do I ask about a parking pass?")).rejects.toThrow(/\[route:\]/); }); - it("is what the test-mode resolver hands the route's evaluator, by its block name", () => { + it("is what the test-mode resolver hands best fit's evaluator, by its block name", () => { const resolver = createKitchenSinkTestModelResolver(); expect(resolver.resolveEvaluationModel).toBeTypeOf("function"); - const resolved = resolver.resolveEvaluationModel!("any/model", "mailbox-route"); + const resolved = resolver.resolveEvaluationModel!("any/model", "coordinator-route"); expect(resolved).toBeDefined(); - expect(resolved).toBe(mailboxRouteMock); + expect(resolved).toBe(coordinatorRouteMock); }); }); describe("the wake scenario", () => { - it("answers in text and never calls the post tool, on a routed post or not", () => { + it("answers in text, with no tool call", () => { agentSeatMock.reset(); - const heard = "devuser in support.help: [scenario:wake] is anyone there?"; - const routed = `${heard}\n\nYou were picked to answer this post, and your reply is posted to support.help as you.`; - for (const turn of [heard, routed]) { - const step = agentSeatMock.next([user(turn)]); - expect(step?.toolCalls).toBeUndefined(); - expect(step?.text).toContain("[reply:wake]"); - } + const step = agentSeatMock.next([user("devuser, through support.help: [scenario:wake] is anyone there?")]); + expect(step?.toolCalls).toBeUndefined(); + expect(step?.text).toContain("[reply:wake]"); }); }); diff --git a/apps/kitchen-sink/test/escalate.test.ts b/apps/kitchen-sink/test/escalate.test.ts deleted file mode 100644 index 7972ff1f0d..0000000000 --- a/apps/kitchen-sink/test/escalate.test.ts +++ /dev/null @@ -1,271 +0,0 @@ -/** - * A specialist files a case that needs a person onto `support.help`'s - * `escalations`, through the mailbox's own `fileTask`, and says so. - * - * Every case boots the whole app in test mode (the scripted model, no key) - * and reaches the seat the way the page does, over the app's router. What is - * graded is what the server kept: the seat's conversation, the mailbox's - * requests, and the rows on the board as the team panel reads them. - * - * Checks, by the spec's ids (`specs/issues/FIX-1611/BUSINESS-RULES.md`, V3): - * - * BR-8 `[scenario:needs-a-person]` calls `escalate` once: one row on - * `escalations` carrying the case, through one `fileTask` request on - * `support.help` signed as the seat, and the reply says it filed. - * BR-10 the model names another author, board and mailbox: the row is - * still on `escalations`, in `support.help`, authored by the seat. - * BR-11 past a host that hands work to an external queue, the tool says - * filing is unavailable, the reply says nothing was filed, and no row - * lands. - * A seat that is not a member of `support.help`, which the mailbox would - * refuse, files nothing, sends nothing, and its reply says nothing was - * filed. The members the tool files for are the ones the mailbox's file - * declares: drop one from `ESCALATION_MEMBERS` and that case fails. - */ -import { afterEach, describe, expect, it, vi } from "vitest"; -import type { FlowState } from "@flow-state-dev/engine"; -import { runAction } from "@flow-state-dev/engine"; -import type { FlowInstance } from "@flow-state-dev/core/types"; -import { createResourceClient, createSessionClient } from "@flow-state-dev/client"; - -// Each case boots the whole app afresh, and the first import is cold. -vi.setConfig({ testTimeout: 30_000 }); - -const MAILBOX = "support.help"; -const BOARD = `${MAILBOX}.escalations`; - -type Item = { type?: string; role?: string; content?: unknown; text?: string }; - -afterEach(async () => { - const hmr = globalThis as { __fsdFlowstate?: FlowState }; - await hmr.__fsdFlowstate?.dispose(); - delete hmr.__fsdFlowstate; - vi.unstubAllEnvs(); - vi.restoreAllMocks(); -}); - -async function bootApp() { - vi.resetModules(); - vi.stubEnv("KITCHEN_SINK_TEST_MODE", "1"); - vi.stubEnv("STORE_TYPE", "memory"); - vi.stubEnv("WORKFORCE_ADMIN_TOKENS", ""); - vi.stubEnv("AI_GATEWAY_API_KEY", ""); - vi.stubEnv("GOAL_CONTROL", ""); - const flowstate = (await import("@/fsdev.config")).default as FlowState; - const router = await flowstate.getRouter(); - const fetcher = async (input: RequestInfo | URL, init?: RequestInit) => { - const url = new URL(String(input), "http://kitchen-sink.local"); - const path = url.pathname - .replace(/^\/api\/flows\/?/, "") - .split("/") - .filter((segment) => segment.length > 0) - .map(decodeURIComponent); - const method = (init?.method ?? "GET").toUpperCase() as "GET" | "POST" | "PATCH" | "DELETE"; - return await router[method](new Request(url, init), { params: { path } }); - }; - return { - flowstate, - router, - sessions: createSessionClient({ fetcher }), - resources: createResourceClient({ fetcher }), - }; -} - -type App = Awaited>; - -const token = () => `case-token-${Math.random().toString(36).slice(2, 10)}`; - -function textOf(item: Item): string { - if (typeof item.content === "string") return item.content; - if (Array.isArray(item.content)) { - return item.content.map((part) => (part as { text?: string }).text ?? "").join(""); - } - return item.text ?? JSON.stringify(item); -} - -/** Send a message to a worker the way the panel's composer does, and wait for the run to finish. */ -async function ask(app: App, worker: string, message: string) { - // A conversation with a worker is a session on its flow's one copy, created naming the worker. - const session = await app.sessions.createSession({ flowKind: "agent", userId: "devuser", state: { workerId: worker } }); - const res = await app.router.POST( - new Request(`http://localhost/api/flows/agent/actions/run`, { - method: "POST", - headers: { "content-type": "application/json", accept: "text/event-stream" }, - body: JSON.stringify({ userId: "devuser", sessionId: session.id, input: { message } }), - }), - { params: { path: ["agent", "actions", "run"] } }, - ); - const stream = await res.text(); - const snapshot = await app.sessions.getSessionState(session.id, { includeItems: true }); - const replies = ((snapshot.items ?? []) as Item[]) - .filter((item) => item.type === "message" && item.role === "assistant") - .map(textOf); - return { status: res.status, stream, replies }; -} - -type Row = { goal?: string; assignee?: string }; - -/** Rows on the board carrying `mark`, as the team panel reads them, waiting for the mailbox's own request. */ -async function rowsWith(app: App, mark: string, expectAny = true): Promise { - for (let i = 0; i < (expectAny ? 100 : 15); i++) { - const page = await app.resources.listCollectionItems(MAILBOX, BOARD, {}); - const rows = ((page as { items?: Array<{ clientData?: Row }> }).items ?? []) - .map((row) => row.clientData ?? {}) - .filter((row) => JSON.stringify(row).includes(mark)); - if (rows.length > 0) return rows; - await new Promise((resolve) => setTimeout(resolve, 20)); - } - return []; -} - -type FileRequest = { status: string; actionName?: string; input?: { board?: string; author?: string } }; - -/** `support.help`'s requests carrying `mark`, once they settle. */ -async function fileRequestsWith(app: App, mark: string): Promise { - let found: FileRequest[] = []; - for (let i = 0; i < 100; i++) { - const requests = (await app.sessions.listSessionRequests(MAILBOX)) as unknown as FileRequest[]; - found = requests.filter((request) => JSON.stringify(request).includes(mark)); - if (found.length > 0 && found.every((r) => !["pending", "queued", "in_progress", "running"].includes(r.status))) { - return found; - } - await new Promise((resolve) => setTimeout(resolve, 20)); - } - return found; -} - -describe("V3 · a specialist files what needs a person", () => { - it("files the case onto escalations through the mailbox's fileTask, authored by the seat, and says so (BR-8)", async () => { - const app = await bootApp(); - const mark = token(); - const ran = await ask(app, "support.devices", `[scenario:needs-a-person] ${mark} the charger caught fire`); - - expect(ran.status).toBe(200); - expect(ran.replies).toEqual([expect.stringContaining("[reply:escalated]")]); - const rows = await rowsWith(app, mark); - expect(rows).toHaveLength(1); - expect(rows[0]!.assignee).toBeUndefined(); - const requests = await fileRequestsWith(app, mark); - expect(requests.map((r) => [r.actionName, r.status, r.input?.board, r.input?.author])).toEqual([ - ["fileTask", "completed", "escalations", "support.devices"], - ]); - }); - - it("files as the seat onto escalations whatever author, board and mailbox the model names (BR-10)", async () => { - const app = await bootApp(); - const mark = token(); - // The scripted call adds an author, a board and a mailbox of its own; the - // tool's input is the case alone, so none of them reaches the filing. - await ask(app, "support.accounts", `[scenario:needs-a-person] [forge-author] ${mark} refund me now`); - - expect(await rowsWith(app, mark)).toHaveLength(1); - const requests = await fileRequestsWith(app, mark); - expect(requests.map((r) => [r.actionName, r.input?.board, r.input?.author])).toEqual([ - ["fileTask", "escalations", "support.accounts"], - ]); - }); - - it("files nothing for a worker that is not a member of support.help, and says so", async () => { - const app = await bootApp(); - // The person hires a worker of their own, holding the tool, through the roster flow. - const roster = await app.sessions.createSession({ flowKind: "workforce-roster", userId: "devuser" }); - const hired = await app.router.POST( - new Request(`http://localhost/api/flows/workforce-roster/actions/hire`, { - method: "POST", - headers: { "content-type": "application/json", accept: "text/event-stream" }, - body: JSON.stringify({ - userId: "devuser", - sessionId: roster.id, - input: { id: "support.bo", instructions: "Answer questions.", settings: { tools: ["escalate"] } }, - }), - }), - { params: { path: ["workforce-roster", "actions", "hire"] } }, - ); - expect(hired.status).toBe(200); - const stream = await hired.text(); - expect(stream, "the hire's own answer").not.toMatch(/"status":"failed"|refused/); - const mark = token(); - const ran = await ask(app, "support.bo", `[scenario:needs-a-person] ${mark} escalate me`); - - // What the person sees: a reply that says nothing was filed, never one - // that claims it was. The mailbox would refuse the seat, so nothing is - // sent to it. - expect(ran.replies).toEqual([expect.stringContaining("[reply:unfiled]")]); - expect(ran.replies.join("\n")).not.toContain("[reply:escalated]"); - expect(await fileRequestsWith(app, mark)).toEqual([]); - expect(await rowsWith(app, mark, false)).toEqual([]); - }); - - it("files for the members support.help's MAILBOX.md declares, and no one else", async () => { - const { readMailboxesDirectory } = await import("@flow-state-dev/workforce/loader"); - const { workforceRoot } = await import("@/workforce/hire"); - const { ESCALATION_MAILBOX, ESCALATION_MEMBERS } = await import("@/workforce/blocks/escalate"); - const { mailboxes, errors } = await readMailboxesDirectory(workforceRoot); - expect(errors).toEqual([]); - const declared = mailboxes.find((mailbox) => mailbox.id === ESCALATION_MAILBOX)?.declared.members; - expect(declared).toBeDefined(); - expect([...ESCALATION_MEMBERS].sort()).toEqual([...(declared as string[])].sort()); - }); - - it("says filing is unavailable past an external dispatcher, and files nothing (BR-11)", async () => { - const app = await bootApp(); - const runtime = await app.flowstate.getRuntime(); - const seat = runtime.registry.get("agent") as FlowInstance; - const session = await app.sessions.createSession({ - flowKind: "agent", - userId: "devuser", - state: { workerId: "support.devices" }, - }); - const mark = token(); - - const ran = await runAction({ - flow: seat, - actionName: "run", - input: { message: `[scenario:needs-a-person] ${mark} the charger caught fire` }, - userId: "devuser", - orgId: "kitchen-sink", - sessionId: session.id, - stores: runtime.stores, - runtimeConfig: { - ...runtime.runtimeConfig, - requestHost: { - ...runtime.runtimeConfig.requestHost, - // What the host answers an `id` delivery when its dispatcher hands - // work to an external queue. - dispatchOperation: async () => ({ - notStarted: true, - externalDispatcher: true, - reason: "the effective dispatcher hands work to an external queue", - }), - } as never, - }, - }); - - expect(ran.error).toBeUndefined(); - const toolOutputs = JSON.stringify((ran.items ?? []).filter((item) => (item as Item).type === "tool_output")); - expect(toolOutputs).toContain("unavailable"); - // What the person sees: a reply that says nothing was filed, never one - // that claims it was. - const replies = (ran.items ?? []) - .filter((item) => (item as Item).type === "message" && (item as Item).role === "assistant") - .map((item) => textOf(item as Item)); - expect(replies).toEqual([expect.stringContaining("[reply:unfiled]")]); - expect(replies.join("\n")).not.toContain("[reply:escalated]"); - expect(await fileRequestsWith(app, mark)).toEqual([]); - expect(await rowsWith(app, mark, false)).toEqual([]); - }); -}); - -describe("support.help's escalations, worked from outside a run", () => { - it("offers the board's task tools as mailbox actions (FIX-1629 BR-21)", async () => { - // The mailbox opts in with `boardActions: true`, so the DevTool's Tasks tab - // can cancel, reassign or settle an escalation from its row. - const app = await bootApp(); - const runtime = await app.flowstate.getRuntime(); - const mailbox = runtime.registry.get("mailbox") as FlowInstance & { actions: Record }; - const suffix = BOARD.replace(/\./g, "_"); - for (const tool of ["cancelTask", "updateTask", "assignTask", "completeTask", "failTask", "blockTask"]) { - expect(Object.keys(mailbox.actions)).toContain(`${tool}_${suffix}`); - } - }); -}); diff --git a/apps/kitchen-sink/test/goal-control.test.ts b/apps/kitchen-sink/test/goal-control.test.ts index 70495e799c..ded880588b 100644 --- a/apps/kitchen-sink/test/goal-control.test.ts +++ b/apps/kitchen-sink/test/goal-control.test.ts @@ -15,9 +15,7 @@ import { readdirSync, readFileSync, statSync } from "node:fs"; import { join, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { handler } from "@flow-state-dev/core"; -import { createWorkerInstallation, mailboxPostCapability, type MailboxManifest } from "@flow-state-dev/workforce"; -import { z } from "zod"; +import { createWorkerInstallation, type WorkerManifest } from "@flow-state-dev/workforce"; import { goalControl, pageGoalControl } from "@/lib/goal-control"; @@ -27,7 +25,7 @@ afterEach(() => { describe("goalControl", () => { it("names no control outside test mode, whatever GOAL_CONTROL says", () => { - vi.stubEnv("GOAL_CONTROL", "no-author-filter"); + vi.stubEnv("GOAL_CONTROL", "no-route"); vi.stubEnv("KITCHEN_SINK_TEST_MODE", ""); expect(goalControl()).toBeUndefined(); vi.stubEnv("KITCHEN_SINK_TEST_MODE", "0"); @@ -36,8 +34,8 @@ describe("goalControl", () => { it("names the control in test mode, and none when GOAL_CONTROL is empty", () => { vi.stubEnv("KITCHEN_SINK_TEST_MODE", "1"); - vi.stubEnv("GOAL_CONTROL", "no-author-filter"); - expect(goalControl()).toBe("no-author-filter"); + vi.stubEnv("GOAL_CONTROL", "no-route"); + expect(goalControl()).toBe("no-route"); vi.stubEnv("GOAL_CONTROL", ""); expect(goalControl()).toBeUndefined(); }); @@ -82,17 +80,17 @@ describe("V4 · every control is read through the gate, and only in this app (BR }); /** Each control, off and on: what it hands back outside test mode, and in it. */ - const wake = handler({ name: "a-wake", inputSchema: z.unknown(), outputSchema: z.unknown(), execute: () => null }); - const catalog = { escalate: handler({ name: "escalate", inputSchema: z.unknown(), outputSchema: z.unknown(), execute: () => null }) }; - const routed = [{ id: "support.help", declared: { routing: { fallback: "support.general" } } }] as unknown as MailboxManifest[]; + const catalog = {}; + const workers = [ + { id: "support.help", declared: { flow: "coordinator", routing: "best-fit", delegates: ["support.general"] }, body: "", skills: [] }, + ] as unknown as WorkerManifest[]; + const flows = [{ kind: "agent" }]; const controls: Array<[name: string, apply: () => Promise<{ swapped: boolean }>]> = [ - ["no-landing", async () => ({ swapped: (await import("@/lib/mailbox-landing-control")).mailboxLandingControl(catalog, mailboxPostCapability, createWorkerInstallation()) !== undefined })], - ["no-filing", async () => ({ swapped: (await import("@/lib/escalate-control")).escalateControl(catalog) !== undefined })], - ["no-route", async () => ({ swapped: (await import("@/lib/mailbox-route-control")).withMailboxRouteControl(routed) !== routed })], - ["name-only-notify", async () => ({ swapped: (await import("@/lib/mailbox-wake-control")).withMailboxWakeControl(wake) !== wake })], - ["no-author-filter", async () => ({ swapped: (await import("@/lib/mailbox-wake-control")).withMailboxWakeControl(wake) !== wake })], - ["post-without-author", async () => ({ swapped: (await import("@/lib/mailbox-post-control")).mailboxPostControl() !== undefined })], + ["no-landing", async () => ({ swapped: (await import("@/lib/landing-control")).landingControl(catalog, createWorkerInstallation()) !== undefined })], + ["no-route", async () => ({ swapped: (await import("@/lib/coordinator-control")).coordinatorWorkersUnderControl(workers) !== workers })], + ["answers-go-on", async () => ({ swapped: (await import("@/lib/coordinator-control")).coordinatorWorkersUnderControl(workers) !== workers })], + ["no-delivery", async () => ({ swapped: (await import("@/lib/coordinator-control")).delegateFlowsUnderControl(flows) !== flows })], ]; it.each(controls)("%s: inert outside test mode, and swapped in under it", async (name, apply) => { diff --git a/apps/kitchen-sink/test/mailbox-reply.test.ts b/apps/kitchen-sink/test/mailbox-reply.test.ts index 30170bc221..66bcfd48c3 100644 --- a/apps/kitchen-sink/test/mailbox-reply.test.ts +++ b/apps/kitchen-sink/test/mailbox-reply.test.ts @@ -1,28 +1,28 @@ /** - * A specialist routed a post answers in the mailbox through its post tool, - * under its own name, once, and its line wakes nobody, on this app's own - * wiring: the real config, the hired roster, and the scripted model and route. + * A specialist best fit picked answers in the person's conversation with + * `support.help`, under its own name, once, and its answer wakes nobody, on + * this app's own wiring: the real config, the hired roster, and the scripted + * model and best-fit evaluation. * - * The line is read the way the page reads it (the mailbox's `mailbox-post` - * items), and the seats' conversations the way the rail lists them (dispatch - * runs included), after the answers settle. + * The line is read the way the page reads it (the conversation's kept + * messages), and the specialists' sessions the way the rail lists them + * (dispatch runs included), after the answers settle. * * Checks, by the spec's ids (`specs/issues/FIX-1594/PLAN.md`, V2 and V3), - * re-pointed onto the routed roster by FIX-1611 (BR-6): - * BR-2 a post carrying `[scenario:reply-in-mailbox]`, routed to - * `support.devices`, wakes it, and its scripted `post-to-mailbox` - * call lands one line in `support.help` authored `support.devices`. - * The tool's line is the answer, so no second line lands (FIX-1610 - * BR-10). - * BR-15 the seat's own conversation holds the tool call and its own reply, - * never a copy of the line. - * BR-9 the line runs no seat: the specialist heard the token once, in the - * person's post, and the others never. + * re-pointed onto the routed roster by FIX-1611 (BR-6) and onto the + * coordinator by FIX-1792 (BR-8): + * BR-2 a post carrying `[scenario:reply-in-mailbox]`, picked for + * `support.devices`, runs it, and its answer lands as one line in + * the person's conversation under `support.devices`. + * BR-9 the line runs no specialist: the specialist heard the token once, + * in the person's post, and the others never. * - * These run on the default wiring only. The controls' red states - * (`GOAL_CONTROL=no-author-filter`, `GOAL_CONTROL=post-without-author`) are - * graded in `goals/kitchen-sink-talk/agent-replies-in-the-mailbox/run.mts` and - * the talk Playwright case, not here. + * BR-15 (the seat's own conversation holds the tool call, never a copy of the + * line) left with the post tool: a delegate's answer is its turn's own reply, + * which its session keeps. + * + * These run on the default wiring only. The controls' red states are graded + * in `goals/kitchen-sink-talk/agent-replies-in-the-mailbox/run.mts`, not here. */ import { afterEach, describe, expect, it, vi } from "vitest"; import type { FlowState } from "@flow-state-dev/engine"; @@ -33,7 +33,7 @@ vi.setConfig({ testTimeout: 60_000 }); type Router = Awaited>; const USER = "devuser"; -const MAILBOX = "support.help"; +const COORDINATOR = "support.help"; const SEAT = "support.devices"; const OTHERS = ["support.accounts", "support.fsd", "support.general"]; const LINE_MARKER = "[reply:in-mailbox]"; @@ -67,25 +67,21 @@ async function call(router: Router, method: "GET" | "POST", segments: string[], return { status: res.status, text: await res.text() }; } -type Line = { author?: string; principal?: string; authorVerified: boolean; body: string }; +type Line = { role?: string; agentName?: string; text: string }; -/** The mailbox's lines as the page reads them: its session's `mailbox-post` items. */ +/** The conversation's lines as the page reads them: its session's kept messages. */ async function linesOf(router: Router, sessionId: string): Promise { - const res = await call(router, "GET", ["sessions", sessionId, "state"], undefined, "?include_items=true&item_types=component&limit=1000"); + const res = await call(router, "GET", ["sessions", sessionId, "state"], undefined, "?include_items=true&item_types=message&limit=1000"); expect(res.status, res.text).toBe(200); - const items = (JSON.parse(res.text) as { items?: Array<{ component?: string; data?: Line }> }).items ?? []; - return items.filter((item) => item.component === "mailbox-post").map((item) => item.data!); + const items = (JSON.parse(res.text) as { items?: Array<{ role?: string; agentName?: string; transient?: boolean; content?: Array<{ text?: string }> }> }).items ?? []; + return items + .filter((item) => item.transient !== true) + .map((item) => ({ role: item.role, agentName: item.agentName, text: (item.content ?? []).map((c) => c.text ?? "").join("") })); } -type Item = { - type?: string; - role?: string; - transient?: boolean; - content?: Array<{ text?: string }>; - toolCall?: { name?: string; arguments?: string }; -}; +type Item = { type?: string; role?: string; transient?: boolean; content?: Array<{ text?: string }> }; -/** Every conversation of a worker (its sessions on `agent`'s one copy) that mentions `token`, with its items. */ +/** Every session of a worker (its sessions on `agent`'s one copy) that mentions `token`, with its items. */ async function conversationsHolding(router: Router, seat: string, token: string): Promise { const listed = await call(router, "GET", ["sessions"], undefined, `?flowId=agent&state.workerId=${encodeURIComponent(seat)}&userId=${USER}&include=dispatch-runs&limit=100`); const rows = (JSON.parse(listed.text) as { sessions: Array<{ id: string }> }).sessions; @@ -114,43 +110,40 @@ const token = () => `reply-token-${Date.now().toString(36)}${Math.random().toStr /** The post each case sends, routed to the specialist. */ const postBody = (mark: string) => `[route:${SEAT}] [scenario:reply-in-mailbox] ${mark} when do refunds post?`; -/** Post as the page does, wait for the specialist's line and answer, and give a wrongly woken seat time to run. */ -async function postAndSettle(router: Router, mark: string): Promise { - const res = await call(router, "POST", ["mailbox", "actions", "post"], { +/** Post as the page does, wait for the specialist's line, and give a wrongly woken specialist time to run. */ +async function postAndSettle(router: Router, mark: string): Promise { + const opened = await call(router, "POST", ["coordinator", "sessions"], { userId: USER, state: { workerId: COORDINATOR } }); + expect(opened.status, opened.text).toBe(201); + const conversation = (JSON.parse(opened.text) as { session: { id: string } }).session.id; + const res = await call(router, "POST", ["coordinator", "actions", "run"], { userId: USER, - sessionId: MAILBOX, - input: { body: postBody(mark) }, + sessionId: conversation, + input: { message: postBody(mark) }, }); expect(res.status, res.text).toBe(200); - await until(async () => (await linesOf(router, MAILBOX)).some((l) => l.body.includes(LINE_MARKER) && l.body.includes(mark)), "the specialist's line"); - await until(async () => (await conversationsHolding(router, SEAT, mark)).length > 0, `${SEAT}'s answer`); + await until(async () => (await linesOf(router, conversation)).some((l) => l.agentName !== undefined && l.text.includes(mark)), "the specialist's line"); await new Promise((resolve) => setTimeout(resolve, 750)); + return conversation; } -describe("V2 · a woken agent seat answers in the mailbox, under its own name", () => { - it("lands one line in support.help as support.devices, and keeps only the tool call in its conversation", async () => { +describe("V2 · a specialist answers in the person's conversation, under its own name", () => { + it("lands one line in the conversation as support.devices, carrying the post's token", async () => { const router = await bootApp(); const mark = token(); - await postAndSettle(router, mark); + const conversation = await postAndSettle(router, mark); - const lines = await linesOf(router, MAILBOX); - const at = lines.findIndex((l) => l.author === undefined && l.body.includes(mark)); - // One line after the person's: the tool's. The routed turn's reply does not land a second. + const lines = await linesOf(router, conversation); + const at = lines.findIndex((l) => l.role === "user" && l.text.includes(mark)); + // One line after the person's: the specialist's answer. expect(lines.slice(at + 1)).toHaveLength(1); const [reply] = lines.slice(at + 1); - expect(reply!.body).toContain(LINE_MARKER); - expect(reply).toMatchObject({ author: SEAT, principal: USER }); - - const [seat] = await conversationsHolding(router, SEAT, mark); - const calls = seat!.filter((i) => i.type === "tool_output" && i.toolCall?.name === "post-to-mailbox"); - expect(calls).toHaveLength(1); - expect(JSON.parse(calls[0]!.toolCall!.arguments!)).toMatchObject({ mailbox: MAILBOX }); - // The line itself is the mailbox's; the seat's conversation holds no copy of it. - expect(keptMessages(seat!).map(textOf).filter((t) => t.includes(reply!.body))).toEqual([]); + expect(reply!.text).toContain(LINE_MARKER); + expect(reply!.text).toContain(mark); + expect(reply).toMatchObject({ agentName: SEAT }); }); }); -describe("V3 · a seat's line wakes nobody", () => { +describe("V3 · a specialist's line wakes nobody", () => { it("the specialist hears the token once, in the person's post, and nobody hears its line", async () => { const router = await bootApp(); const mark = token(); @@ -161,10 +154,10 @@ describe("V3 · a seat's line wakes nobody", () => { .filter((m) => m.role === "user" && textOf(m).includes(mark)) .map(textOf); expect(heard, `${SEAT}'s turns carrying the token`).toEqual([ - expect.stringContaining(`${USER} in ${MAILBOX}: ${postBody(mark)}`), + expect.stringContaining(`${USER}, through ${COORDINATOR}: ${postBody(mark)}`), ]); for (const other of OTHERS) { - expect(await conversationsHolding(router, other, mark), `${other}'s conversations carrying the token`).toEqual([]); + expect(await conversationsHolding(router, other, mark), `${other}'s sessions carrying the token`).toEqual([]); } }); }); diff --git a/apps/kitchen-sink/test/mailbox-talk.test.ts b/apps/kitchen-sink/test/mailbox-talk.test.ts index 97738013e4..a65fba0ce6 100644 --- a/apps/kitchen-sink/test/mailbox-talk.test.ts +++ b/apps/kitchen-sink/test/mailbox-talk.test.ts @@ -1,23 +1,21 @@ /** - * Posting to a mailbox the way the page does, through the app's real config. + * Posting to `support.help` the way the page does, through the app's real config. * - * The page posts with `{ body }` and no `author`, because the person at it is - * not a member. What it then shows is the mailbox's `mailbox-post` items, read + * The page opens the person's conversation with the coordinator (a session on + * `coordinator` naming `support.help`) and posts through its door with the + * post alone. What it then shows is the conversation's kept messages, read * from the session like any conversation. This case holds what that promises - * on this app's own wiring: the line lands once, it names the server's - * principal, and a specialist answers it. + * on this app's own wiring: the post is kept once, as the person's turn, and a + * specialist answers it under its own name. * * Checks, by the spec's ids (`specs/issues/FIX-1585/PLAN.md`), re-pointed onto - * `support.help` by FIX-1611: + * `support.help` by FIX-1611 and onto the coordinator by FIX-1792 (BR-8): * - * V3 A post with no author lands on `support.help` once, labelled with the - * server's principal (`devuser`). It names no specialist, so the scripted - * route's call fails and `support.general`, the mailbox's fallback, - * answers it under its own name (FIX-1611 BR-5). (The built-in kind's own - * red is `packages/workforce`'s `mailbox-post-items.test.ts`.) - * - * V4, the `digest` kind's post, left with the kind: this app has no mailbox - * kind of its own. + * V3 A post lands on the person's conversation with `support.help` once, as + * a `user` message. It names no specialist, so the scripted best-fit call + * fails and `support.general`, the coordinator's fallback, answers it + * under its own name. (The coordinator's own red is `packages/workforce`'s + * coordinator suite.) */ import { afterEach, describe, expect, it, vi } from "vitest"; import type { FlowState } from "@flow-state-dev/engine"; @@ -55,24 +53,33 @@ async function call(router: Router, method: "GET" | "POST", segments: string[], return { status: res.status, text: await res.text() }; } -/** Post the way the page's composer does: the mailbox's own action, body only. */ -async function post(router: Router, flowId: string, sessionId: string, body: string) { - const res = await call(router, "POST", [flowId, "actions", "post"], { userId: "devuser", sessionId, input: { body } }); +/** The person's conversation with `support.help`, opened as a session on `coordinator` naming it. */ +async function openConversation(router: Router): Promise { + const res = await call(router, "POST", ["coordinator", "sessions"], { userId: "devuser", state: { workerId: "support.help" } }); + expect(res.status, res.text).toBe(201); + return (JSON.parse(res.text) as { session: { id: string } }).session.id; +} + +/** Post the way the page's composer does: the coordinator's door, the post only. */ +async function post(router: Router, sessionId: string, message: string) { + const res = await call(router, "POST", ["coordinator", "actions", "run"], { userId: "devuser", sessionId, input: { message } }); expect(res.status, res.text).toBe(200); return res; } -type Line = { id: string; body: string; principal?: string; author?: string }; +type Line = { role?: string; agentName?: string; text: string }; -/** The mailbox's lines as the page reads them: its session's `mailbox-post` items. */ +/** The conversation's lines as the page reads them: its session's kept messages. */ async function linesOf(router: Router, sessionId: string): Promise { - const res = await call(router, "GET", ["sessions", sessionId, "state"], undefined, "?include_items=true&item_types=component&limit=1000"); + const res = await call(router, "GET", ["sessions", sessionId, "state"], undefined, "?include_items=true&item_types=message&limit=1000"); expect(res.status, res.text).toBe(200); - const items = (JSON.parse(res.text) as { items?: Array<{ component?: string; data?: Line }> }).items ?? []; - return items.filter((item) => item.component === "mailbox-post").map((item) => item.data!); + const items = (JSON.parse(res.text) as { items?: Array<{ role?: string; agentName?: string; transient?: boolean; content?: Array<{ text?: string }> }> }).items ?? []; + return items + .filter((item) => item.transient !== true) + .map((item) => ({ role: item.role, agentName: item.agentName, text: (item.content ?? []).map((c) => c.text ?? "").join("") })); } -/** Wait for the post's separate fan-out request to finish. */ +/** Wait for the delegate's answer to land. */ async function until(predicate: () => Promise, label: string): Promise { for (let i = 0; i < 300; i++) { if (await predicate()) return; @@ -81,25 +88,26 @@ async function until(predicate: () => Promise, label: string): Promise< throw new Error(`timed out waiting for ${label}`); } -describe("V3 · a post from the page, on the app's own mailbox", () => { - it("lands on support.help once, as devuser, and the fallback specialist answers it", async () => { +describe("V3 · a post from the page, on the app's own coordinator", () => { + it("is kept once, as the person's turn, and the fallback specialist answers it under its own name", async () => { const router = await bootApp(); + const conversation = await openConversation(router); const text = `same line ${Date.now()}`; - await post(router, "mailbox", "support.help", text); + await post(router, conversation, text); - const mine = (await linesOf(router, "support.help")).filter((line) => line.body === text); + const mine = (await linesOf(router, conversation)).filter((line) => line.text === text); expect(mine).toHaveLength(1); - expect(mine[0]?.principal).toBe("devuser"); - expect(mine[0]?.author).toBeUndefined(); + expect(mine[0]?.role).toBe("user"); + expect(mine[0]?.agentName).toBeUndefined(); - // The route cannot pick anyone for a post that names nobody, so the + // Best fit cannot pick anyone for a post that names nobody, so the // fallback answers, as one line under its own name. const answers = async () => { - const lines = await linesOf(router, "support.help"); - return lines.slice(lines.findIndex((line) => line.body === text) + 1); + const lines = await linesOf(router, conversation); + return lines.slice(lines.findIndex((line) => line.text === text) + 1); }; await until(async () => (await answers()).length > 0, "the fallback's line"); - expect((await answers()).map((line) => line.author)).toEqual(["support.general"]); + expect((await answers()).map((line) => line.agentName)).toEqual(["support.general"]); }); }); diff --git a/apps/kitchen-sink/test/mailbox-wake.test.ts b/apps/kitchen-sink/test/mailbox-wake.test.ts index dbf8882d38..9ebe22b159 100644 --- a/apps/kitchen-sink/test/mailbox-wake.test.ts +++ b/apps/kitchen-sink/test/mailbox-wake.test.ts @@ -1,53 +1,42 @@ /** - * A post to `support.help` runs the specialist the route picked once, on this + * A post to `support.help` runs the specialist best fit picked once, on this * app's own wiring: the real config, the hired roster, and the scripted model - * and route. + * and best-fit evaluation. * - * What is graded is always the seats' own conversations, read back through the - * same routes the page reads, never the fan-out's output. A seat's mailbox - * conversation is a dispatch run of the mailbox, so it is listed only with - * dispatch runs included. + * What is graded is always the specialists' own conversations, read back + * through the same routes the page reads, never the coordinator's records. A + * specialist's session for the person's conversation with `support.help` is a + * dispatch run of that conversation, so it is listed only with dispatch runs + * included. * * Checks, by the spec's ids (`specs/issues/FIX-1590/BUSINESS-RULES.md`, V3), - * re-pointed onto the routed roster by FIX-1611, and the red state each was - * seen in before its green was trusted: + * re-pointed onto the routed roster by FIX-1611 and onto the coordinator by + * FIX-1792 (BR-8): * - * BR-1 a post with no author runs the specialist it was routed to once, in - * one conversation of its own, the post heard once with one scripted - * reply under it, and no other specialist hears it (FIX-1611 BR-4). - * Red: the name-only stub (`GOAL_CONTROL=name-only-notify`) runs - * nobody; the route taken off (`GOAL_CONTROL=no-route`) runs everyone. - * BR-3 a post a seat wrote, through the internal seat entry, runs no seat, - * the writer included (FIX-1602's BR-3). A public post that only - * claims that seat as `author` still runs the specialist. Red: the - * seat mark stripped before the wake (`GOAL_CONTROL=no-author-filter`). - * BR-9 a second post lands in the same conversation of the specialist. - * BR-10 two posts at once each run exactly once, in that same conversation. - * BR-14 the post's own request carries no seat's answer: the wake runs in - * the mailbox's hand-off request. - * BR-11, BR-12 need a roster this app does not have (a seat in two - * mailboxes, a refused wake), so they are held on the factory below, - * with a roster built for them. + * BR-1 a post runs the specialist best fit picked once, in one session of + * its own under the person's conversation, the post heard once with + * one scripted reply under it, and no other specialist hears it + * (FIX-1611 BR-4). Red: no flow taking a delegated post + * (`GOAL_CONTROL=no-delivery`) runs nobody; best fit read as + * `everyone` (`GOAL_CONTROL=no-route`) runs everyone. + * BR-3 a specialist's answer runs no specialist, its writer included. + * Red: the coordinator read with `rounds: 1` + * (`GOAL_CONTROL=answers-go-on`). + * BR-9 a second post lands in the same session of the specialist. + * BR-10 two posts at once each run exactly once, in that same session. + * BR-14 the post's own request carries no specialist's answer: the answer + * lands in a request of its own. * - * BR-2 and BR-6 (the name-only line for a member the wake does not run) and - * BR-4 (a mailbox with no agent member) left with the roster that had such - * members: every member of `support.help` is an agent, so the notify has no - * name-only line to give. FIX-1602's fixture host still proves the wake - * without one. + * A post that claims an author is refused at the door: the coordinator takes + * the post alone, so there is no author to claim. BR-11 and BR-12 (a seat in + * two mailboxes, a refused wake) left with the mailbox's wake; the + * coordinator's own suite in `packages/workforce` holds a delegate's session + * per conversation and a delivery that fails. */ import { afterEach, describe, expect, it, vi } from "vitest"; -import { DEFAULT_ORG_ID, defineFlow, handler } from "@flow-state-dev/core"; -import { z } from "zod"; -import type { FlowState, StoreRegistry } from "@flow-state-dev/engine"; -import { createFlowState, inMemoryStores, runAction } from "@flow-state-dev/engine"; -import { - MAILBOX_KIND, - createWorkerInstallation, - mailboxNotifyInputSchema, - defineMailboxFlow, - hireWorkforce, - workerConfigSchema, -} from "@flow-state-dev/workforce"; +import type { FlowState } from "@flow-state-dev/engine"; + +import { delegateOfRun } from "@/lib/workforce-shell"; // Each case boots the whole app afresh, and the first import is cold. vi.setConfig({ testTimeout: 60_000 }); @@ -55,9 +44,9 @@ vi.setConfig({ testTimeout: 60_000 }); type Router = Awaited>; const USER = "devuser"; -const MAILBOX = "support.help"; +const COORDINATOR = "support.help"; const SPECIALISTS = ["support.devices", "support.accounts", "support.fsd", "support.general"]; -/** The specialist each post names with `[route:]`, which the scripted route honours. */ +/** The specialist each post names with `[route:]`, which the scripted evaluation honours. */ const ROUTED = "support.accounts"; const OTHERS = SPECIALISTS.filter((seat) => seat !== ROUTED); @@ -90,48 +79,26 @@ async function call(router: Router, method: "GET" | "POST", segments: string[], return { status: res.status, text: await res.text() }; } -/** - * Post the way the page's composer does. `author`, when set, is only the - * caller's claim. It is not a seat's post. - */ -async function post(router: Router, sessionId: string, body: string, author?: string) { - const input = author === undefined ? { body } : { body, author }; - const res = await call(router, "POST", ["mailbox", "actions", "post"], { userId: USER, sessionId, input }); - expect(res.status, res.text).toBe(200); - return res; +/** The person's conversation with `support.help`, opened as a session on `coordinator` naming it. */ +async function openConversation(router: Router): Promise { + const res = await call(router, "POST", ["coordinator", "sessions"], { userId: USER, state: { workerId: COORDINATOR } }); + expect(res.status, res.text).toBe(201); + return (JSON.parse(res.text) as { session: { id: string } }).session.id; } -/** - * Post the way a seat does: the mailbox's `seatPost` action, which is what - * marks the line a seat's. `author` is the name on the line. - */ -async function seatPost(sessionId: string, body: string, author: string) { - const flowstate = (globalThis as { __fsdFlowstate?: FlowState }).__fsdFlowstate; - if (flowstate === undefined) throw new Error("the app is not booted"); - const runtime = await flowstate.getRuntime(); - const mailbox = runtime.registry.get(MAILBOX_KIND); - if (mailbox === undefined) throw new Error("the mailbox kind is not registered"); - const { KITCHEN_SINK_ORG_ID } = await import("@/lib/kitchen-sink-principal"); - const result = await runAction({ - source: "internal", - orgId: KITCHEN_SINK_ORG_ID, - flow: mailbox, - actionName: "seatPost", - input: { body, author }, - userId: USER, - sessionId, - stores: runtime.stores, - runtimeConfig: { ...runtime.runtimeConfig }, - }); - expect(result.error, JSON.stringify(result.error)).toBeUndefined(); +/** Post the way the page's composer does: the coordinator's door, the post only. */ +async function post(router: Router, sessionId: string, message: string) { + const res = await call(router, "POST", ["coordinator", "actions", "run"], { userId: USER, sessionId, input: { message } }); + expect(res.status, res.text).toBe(200); + return res; } type Message = { role: string; text: string }; type Conversation = { sessionId: string; parentSessionId?: string; messages: Message[] }; /** - * Every conversation of a worker, dispatch runs included, with its kept - * messages: the sessions on `agent`'s one copy created naming the worker. + * Every session of a worker, dispatch runs included, with its kept messages: + * the sessions on `agent`'s one copy created naming the worker. */ async function conversationsOf(router: Router, seat: string): Promise { const listed = await call( @@ -158,7 +125,7 @@ async function conversationsOf(router: Router, seat: string): Promise { return (await conversationsOf(router, seat)).filter((c) => c.messages.some((m) => m.text.includes(token))); } @@ -181,39 +148,47 @@ async function routedAnswered(router: Router, token: string): Promise { const token = (label: string) => `${label}-${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`; -describe("V3 · a post runs the specialist it was routed to once, on the app's own mailbox", () => { - it("runs support.accounts once, in a conversation of its own, and nobody else", async () => { +describe("V3 · a post runs the specialist best fit picked once, on the app's own coordinator", () => { + it("runs support.accounts once, in a session of its own under the person's conversation, and nobody else", async () => { const router = await bootApp(); + const conversation = await openConversation(router); const mark = token("wake"); const body = `[route:${ROUTED}] [scenario:wake] ${mark} can someone look at my refund?`; - const posted = await post(router, MAILBOX, body); + const posted = await post(router, conversation, body); await until(() => routedAnswered(router, mark), "the routed specialist to answer"); // Give a wrongly woken specialist time to answer, so its absence is not a race. await new Promise((resolve) => setTimeout(resolve, 500)); const convs = await holding(router, ROUTED, mark); - expect(convs, `${ROUTED}'s conversations holding the post`).toHaveLength(1); + expect(convs, `${ROUTED}'s sessions holding the post`).toHaveLength(1); const [conv] = convs; - // A run the mailbox started, not a conversation the person opened. - expect(conv!.parentSessionId).toBe(MAILBOX); + // A run the person's conversation started, not a conversation the person opened. + expect(conv!.parentSessionId).toBe(conversation); expect(conv!.messages).toEqual([ - { role: "user", text: expect.stringContaining(`${USER} in ${MAILBOX}: ${body}`) }, + { role: "user", text: expect.stringContaining(`${USER}, through ${COORDINATOR}: ${body}`) }, { role: "assistant", text: expect.stringContaining("[reply:wake]") }, ]); // Only the routed specialist heard it. for (const seat of OTHERS) expect(await holding(router, seat, mark), seat).toEqual([]); - // BR-14: the post's own request streamed no seat's answer. + // BR-14: the post's own request streamed no specialist's answer. expect(posted.text).not.toContain("[reply:wake]"); + // The run the page lists under the conversation names its delegate by the + // key the page's working row reads (`delegateOfRun`). + const children = await call(router, "GET", ["sessions", conversation, "children"]); + expect(children.status, children.text).toBe(200); + const runs = (JSON.parse(children.text) as { children: Array<{ id: string; topic?: string }> }).children; + expect(runs.filter((run) => run.id === conv!.sessionId).map(delegateOfRun)).toEqual([ROUTED]); }); - it("lands a second post in the same conversation of the specialist", async () => { + it("lands a second post in the same session of the specialist", async () => { const router = await bootApp(); + const conversation = await openConversation(router); const first = token("first"); const second = token("second"); - await post(router, MAILBOX, `[route:${ROUTED}] [scenario:wake] ${first}`); + await post(router, conversation, `[route:${ROUTED}] [scenario:wake] ${first}`); await until(() => routedAnswered(router, first), "the first answer"); - await post(router, MAILBOX, `[route:${ROUTED}] [scenario:wake] ${second}`); + await post(router, conversation, `[route:${ROUTED}] [scenario:wake] ${second}`); await until(() => routedAnswered(router, second), "the second answer"); const a = await holding(router, ROUTED, first); @@ -223,174 +198,56 @@ describe("V3 · a post runs the specialist it was routed to once, on the app's o expect(b[0]!.messages.filter((m) => m.role === "user")).toHaveLength(2); }); - it("runs two posts that arrive together once each, in the one conversation", async () => { + it("runs two posts that arrive together once each, in the one session", async () => { const router = await bootApp(); + const conversation = await openConversation(router); const a = token("together-a"); const b = token("together-b"); await Promise.all([ - post(router, MAILBOX, `[route:${ROUTED}] [scenario:wake] ${a}`), - post(router, MAILBOX, `[route:${ROUTED}] [scenario:wake] ${b}`), + post(router, conversation, `[route:${ROUTED}] [scenario:wake] ${a}`), + post(router, conversation, `[route:${ROUTED}] [scenario:wake] ${b}`), ]); await until(async () => (await routedAnswered(router, a)) && (await routedAnswered(router, b)), "both posts answered"); - const convs = (await conversationsOf(router, ROUTED)).filter((c) => c.parentSessionId === MAILBOX); - expect(convs, `${ROUTED}'s mailbox conversations`).toHaveLength(1); + const convs = (await conversationsOf(router, ROUTED)).filter((c) => c.parentSessionId === conversation); + expect(convs, `${ROUTED}'s sessions under the conversation`).toHaveLength(1); const turns = convs[0]!.messages.filter((m) => m.role === "user").map((m) => m.text); // Each post heard exactly once, in no guaranteed order. expect(turns.filter((t) => t.includes(a))).toHaveLength(1); expect(turns.filter((t) => t.includes(b))).toHaveLength(1); }); - it("runs no seat on a post a seat wrote, the writer included", async () => { + it("runs no specialist on a specialist's answer, the writer included", async () => { const router = await bootApp(); - const mark = token("authored"); - await seatPost(MAILBOX, `[route:${ROUTED}] [scenario:wake] ${mark}`, "support.devices"); - // A wrongly woken seat answers within this in every run of the scripted model. + const conversation = await openConversation(router); + const mark = token("answered"); + await post(router, conversation, `[route:${ROUTED}] [scenario:wake] ${mark}`); + await until(() => routedAnswered(router, mark), "the routed specialist to answer"); + // A wrongly woken specialist answers within this in every run of the scripted model. await new Promise((resolve) => setTimeout(resolve, 1_500)); - for (const seat of SPECIALISTS) expect(await holding(router, seat, mark), seat).toEqual([]); + // The answer is "[reply:wake] …": nobody was handed it as a turn. + for (const seat of SPECIALISTS) { + const heardAnswer = (await conversationsOf(router, seat)) + .flatMap((c) => c.messages) + .filter((m) => m.role === "user" && m.text.includes("[reply:wake]")); + expect(heardAnswer, seat).toEqual([]); + } }); - it("still runs the routed specialist when a public post claims a hire address as author", async () => { + it("refuses a post that claims an author, and runs nobody on it", async () => { const router = await bootApp(); + const conversation = await openConversation(router); const mark = token("claimed"); - const body = `[route:${ROUTED}] [scenario:wake] ${mark} can someone look at my refund?`; - await post(router, MAILBOX, body, "support.devices"); - - await until(() => routedAnswered(router, mark), "the routed specialist to answer a claimed author"); - const convs = await holding(router, ROUTED, mark); - expect(convs, `${ROUTED}'s conversations holding the claimed post`).toHaveLength(1); - expect(convs[0]!.messages[0]?.text).toEqual(expect.stringContaining(`support.devices in ${MAILBOX}: ${body}`)); - }); -}); - -// --------------------------------------------------------------------------- -// The factory, on a roster built for the cases the app's own cannot reach. -// --------------------------------------------------------------------------- - -async function bind(stores: StoreRegistry, sessionId: string, members: string[]): Promise { - const now = Date.now(); - await stores.session.set( - sessionId, - { - id: sessionId, - flowKind: MAILBOX_KIND, - flowId: MAILBOX_KIND, + const res = await call(router, "POST", ["coordinator", "actions", "run"], { userId: USER, - orgId: DEFAULT_ORG_ID, - state: { members, instructions: "Charter.", transcript: [] }, - lineageId: `lin_${sessionId}`, - version: 0, - createdAt: now, - updatedAt: now, - journal: [], - } as never, - "any", - ); -} - -describe("V3 · the wake factory, off the app's own roster", () => { - /** - * `support.otto` is a standard worker on `agent`, whose copy is registered. - * `support.iris` runs on `listener`, whose copy is handed to the wake but not - * registered, so its dispatch is refused. `support.lost` never loaded, so - * there is no worker to wake. - */ - async function host() { - vi.resetModules(); - vi.stubEnv("KITCHEN_SINK_TEST_MODE", "1"); - vi.stubEnv("GOAL_CONTROL", ""); - const { notifyFor } = await import("@/workforce/mailbox-notify"); - const { createKitchenSinkTestModelResolver } = await import("@/test/mock-flowstate"); - let flows: Record = {}; - const installation = createWorkerInstallation({ - standardWorkers: [ - { id: "support.iris", declared: { flow: "listener" }, body: "You answer questions." }, - { id: "support.otto", declared: {}, body: "You answer questions." }, - ], - workerFlows: () => flows as never, + sessionId: conversation, + input: { message: `[route:${ROUTED}] [scenario:wake] ${mark}`, author: "support.devices" }, }); - const quiet = handler({ name: "listener-heard", inputSchema: z.unknown(), outputSchema: z.unknown(), execute: () => null }); - const door = z.object({ message: z.string() }); - flows = { - listener: defineFlow({ - kind: "listener", - configSchema: workerConfigSchema(), - session: installation.session(), - resources: { ...installation.resources }, - actions: { run: { inputSchema: door, userMessage: (i: { message: string }) => i.message, block: quiet } }, - internal: { actions: { onMailboxPost: { inputSchema: mailboxNotifyInputSchema, block: quiet } } }, - }), - }; - const copies = hireWorkforce(installation); - const agent = copies.find((copy) => copy.id === "agent")!; - const mailbox = defineMailboxFlow({ notify: notifyFor(copies, installation) })(); - const state = createFlowState({ - flows: { [MAILBOX_KIND]: mailbox, [agent.id]: agent }, - stores: { default: { primary: inMemoryStores() } }, - modelResolver: createKitchenSinkTestModelResolver(), - }); - const runtime = await state.getRuntime(); - const send = (sessionId: string, body: string) => - runAction({ - orgId: DEFAULT_ORG_ID, - flow: mailbox, - actionName: "post", - input: { body }, - userId: USER, - sessionId, - stores: runtime.stores, - runtimeConfig: { ...runtime.runtimeConfig }, - }); - const ottoRuns = async () => { - const sessions = await runtime.stores.session.list({ flowId: "agent", parentage: "all" }); - return sessions.filter((s) => s.parentSessionId != null && (s.state as { workerId?: string }).workerId === "support.otto"); - }; - return { state, runtime, send, ottoRuns }; - } - - it("still runs the other members when one wake is refused, or has no seat", async () => { - const { state, runtime, send, ottoRuns } = await host(); - try { - await bind(runtime.stores, "room.one", ["support.iris", "support.otto", "support.lost"]); - const sent = await send("room.one", "[scenario:wake] refused-one"); - expect(sent.error).toBeUndefined(); - - await until(async () => (await ottoRuns()).length === 1, "otto's run"); - const mailboxItems = async () => - JSON.stringify(await runtime.stores.request.list({ sessionId: "room.one", withItems: true })); - // BR-12: iris's refusal is recorded in the fan-out's own request. - await until(async () => (await mailboxItems()).includes("flow-not-found"), "iris's refusal"); - const items = await mailboxItems(); - // The post stays written, whatever the wakes did. - expect(items).toContain('"component":"mailbox-post"'); - expect(items).toContain("refused-one"); - } finally { - await state.dispose(); - } - }); - - it("keeps one conversation per mailbox for a seat in two mailboxes", async () => { - const { state, runtime, send, ottoRuns } = await host(); - try { - await bind(runtime.stores, "room.one", ["support.otto"]); - await bind(runtime.stores, "room.two", ["support.otto"]); - await send("room.one", "[scenario:wake] in room one"); - await send("room.two", "[scenario:wake] in room two"); - await until(async () => (await ottoRuns()).length === 2, "a conversation per mailbox"); - - const runs = await ottoRuns(); - expect(runs.map((r) => r.parentSessionId).sort()).toEqual(["room.one", "room.two"]); - for (const run of runs) { - const [request] = await runtime.stores.request.list({ sessionId: run.id, withItems: true }); - const text = JSON.stringify(request?.items ?? []); - // Neither sees the other's post. - const other = run.parentSessionId === "room.one" ? "room two" : "room one"; - expect(text).not.toContain(other); - } - } finally { - await state.dispose(); - } + // Refused in the request's own stream, naming the key the door does not take. + expect(res.text).toContain("request.failed"); + expect(res.text).toContain("Unrecognized key(s) in object: 'author'"); + await new Promise((resolve) => setTimeout(resolve, 500)); + for (const seat of SPECIALISTS) expect(await holding(router, seat, mark), seat).toEqual([]); }); }); - diff --git a/apps/kitchen-sink/test/mock-flowstate.ts b/apps/kitchen-sink/test/mock-flowstate.ts index af14d160af..1d9e011c6d 100644 --- a/apps/kitchen-sink/test/mock-flowstate.ts +++ b/apps/kitchen-sink/test/mock-flowstate.ts @@ -22,7 +22,7 @@ import { autoTitleMock, sideChainBriefMock, agentSeatMock, - mailboxRouteMock, + coordinatorRouteMock, } from "@/lib/e2e-mock-script"; /** Build the mocked model resolver used in `KITCHEN_SINK_TEST_MODE`. */ @@ -37,8 +37,8 @@ export function createKitchenSinkTestModelResolver(): ModelResolver { "agent-answer": agentSeatMock, "agent-answer-with-activate-tool": agentSeatMock, }, - // A routed mailbox's one evaluation, by its evaluator's block name. - evaluators: { "mailbox-route": mailboxRouteMock }, + // A best-fit coordinator's one evaluation, by its evaluator's block name. + evaluators: { "coordinator-route": coordinatorRouteMock }, policy: "allow", }); const held = ((modelId, blockName, options) => { @@ -65,7 +65,7 @@ export function createKitchenSinkTestModelResolver(): ModelResolver { }; }) as ModelResolver; held.resolveId = scripted.resolveId; - // Evaluators resolve through the same resolver: a routed mailbox's route. + // Evaluators resolve through the same resolver: a best-fit coordinator's pick. held.resolveEvaluationModel = scripted.resolveEvaluationModel; return held; } diff --git a/apps/kitchen-sink/test/named-org.test.ts b/apps/kitchen-sink/test/named-org.test.ts index 12c4b86698..7a5386558c 100644 --- a/apps/kitchen-sink/test/named-org.test.ts +++ b/apps/kitchen-sink/test/named-org.test.ts @@ -2,11 +2,10 @@ * Kitchen-sink runs as one named organization — driven through the app's own * `fsdev.config.ts` and the router it builds, not through `runAction`. * - * Every case imports the real config. So the host resolver, the boot's - * mailbox open and the step that clears a pre-change - * store's mailboxes are the ones the app ships, and a request goes through - * route-level authentication exactly as a browser's does. Promoted from the - * spec POC (`specs/issues/FIX-1500/poc/named-org/`). + * Every case imports the real config. So the host resolver is the one the app + * ships, and a request goes through route-level authentication exactly as a + * browser's does. Promoted from the spec POC + * (`specs/issues/FIX-1500/poc/named-org/`). * * One thing is substituted, below the wiring under test: the model. * `KITCHEN_SINK_TEST_MODE=1` makes the config build its model resolver from @@ -16,25 +15,19 @@ * Checks, by the spec's ids (`specs/issues/FIX-1500/PLAN.md`), and the red * state each was seen in before its green was trusted: * - * V18 A session on the assistant's flow, with a worker and on a mailbox - * binds to `kitchen-sink`, whatever the body says. Red: remove `resolvePrincipal` - * from `fsdev.config.ts` — every session binds to `__fsd_default_org__`. - * V19 A store written before the app named its organization is not - * upgraded: it is wiped (the owner's call on #2159). The boot over one - * refuses to start, names every mailbox stored under another - * organization, and says to delete the store. The guard only reads: no - * mailbox is moved, rebound or deleted. Red: remove the guard in `fsdev.config.ts` — the boot fails with the - * bare `mailbox "support.help" could not be opened — Request failed - * (403)`, which names neither the cause nor the fix. + * V18 A session on the assistant's flow, with a worker and with the + * coordinator binds to `kitchen-sink`, whatever the body says + * (FIX-1792 BR-25). Red: remove `resolvePrincipal` from + * `fsdev.config.ts` — every session binds to `__fsd_default_org__`. + * + * V19 (a boot over a store written before the app named its organization + * refuses, naming each stored mailbox) and the refusal of a store written + * before mailboxes were renamed left with the mailboxes the boot opened + * (FIX-1792 S6, D2): the boot opens no session, so it reads none an earlier + * store holds, and nothing old is refused by name. */ import { afterEach, describe, expect, it, vi } from "vitest"; -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { DEFAULT_ORG_ID } from "@flow-state-dev/core"; -import { createFilesystemStores, createFlowState, filesystemStores, type FlowState } from "@flow-state-dev/engine"; -import { createSessionClient } from "@flow-state-dev/client"; -import { PRE_RENAME_NAMES, openMailboxes } from "@flow-state-dev/workforce"; +import type { FlowState } from "@flow-state-dev/engine"; type ScriptStep = | { toolCalls: Array<{ toolCallId: string; toolName: string; args: Record }> } @@ -69,25 +62,20 @@ type Router = Awaited>; // One boot of the app, from its real config. // --------------------------------------------------------------------------- -const cleanups: Array<() => Promise> = []; - afterEach(async () => { const hmr = globalThis as { __fsdFlowstate?: FlowState }; await hmr.__fsdFlowstate?.dispose(); delete hmr.__fsdFlowstate; - while (cleanups.length > 0) await cleanups.pop()!(); vi.unstubAllEnvs(); vi.restoreAllMocks(); }); -async function bootApp(options: { dataDir?: string; steps?: ScriptStep[] } = {}) { +async function bootApp(options: { steps?: ScriptStep[] } = {}) { vi.resetModules(); script.steps = options.steps ?? [{ text: "ok" }]; vi.stubEnv("KITCHEN_SINK_TEST_MODE", "1"); - vi.stubEnv("STORE_TYPE", options.dataDir === undefined ? "memory" : "filesystem"); + vi.stubEnv("STORE_TYPE", "memory"); delete process.env.FSDEV_DEFAULT_MODEL; - // The config roots the filesystem profile at `/.fsdev/data`. - if (options.dataDir !== undefined) vi.spyOn(process, "cwd").mockReturnValue(options.dataDir); const log: string[] = []; const realError = console.error; vi.spyOn(console, "error").mockImplementation((...args: unknown[]) => { @@ -123,10 +111,6 @@ async function call( return { status: res.status, text: await res.text() }; } -/** An out-of-band handle on a filesystem store: not the booted runtime's. */ -const storeAt = (dataDir: string) => - createFilesystemStores({ rootDir: path.join(dataDir, ".fsdev", "data"), developmentOnly: true }); - const json = (text: string) => (text.length > 0 ? JSON.parse(text) : null); /** Open a session the way the page does: a body `userId`, which a resolver overrides. */ @@ -143,151 +127,37 @@ async function act(router: Router, flowId: string, action: string, sessionId: st // --------------------------------------------------------------------------- describe("V18 · one organization, from the host resolver", () => { - it("binds the assistant's flow, a worker's conversation and a mailbox to kitchen-sink, whatever the body says", async () => { + it("binds the assistant's flow, a worker's conversation and a conversation with the coordinator to kitchen-sink, whatever the body says", async () => { const { router } = await bootApp(); const opens: Array<[string, Record]> = [ ["chat-agent", {}], ["agent", { state: { workerId: "support.devices" } }], ["agent", { state: { workerId: "support.general" } }], - ["mailbox", {}], + ["coordinator", { state: { workerId: "support.help" } }], ]; for (const [flowId, body] of opens) { const opened = await openSession(router, flowId, { orgId: "globex", ...body }); expect(opened.status, `${flowId}: ${opened.text}`).toBe(201); expect(opened.orgId, flowId).toBe(ORG); } - // And the mailbox the boot itself opened. - const help = await call(router, "GET", ["sessions", "support.help"]); - expect(help.status, help.text).toBe(200); - expect(json(help.text).session.orgId).toBe(ORG); - }); - -}); - -describe("V19 · a store written before the app named its organization", () => { - /** - * The boot the app ran before it named its organization, for the part that - * matters here: the same mailbox kinds and the same mailbox open, through a - * router with no resolver, so every mailbox session lands in the - * development organization. - */ - async function preChangeBoot(dataDir: string) { - const { buildKitchenSinkWorkforce } = await import("@/workforce/hire"); - const workforce = await buildKitchenSinkWorkforce(); - const flowstate = createFlowState({ - flows: Object.fromEntries(workforce.mailboxFlows.map((flow) => [flow.id, flow])), - stores: { dev: { primary: filesystemStores({ rootDir: path.join(dataDir, ".fsdev", "data") }) } }, - }); - const router = await flowstate.getRouter(); - const client = createSessionClient({ - fetcher: async (input, init) => { - const url = new URL(String(input), "http://kitchen-sink.local"); - const segments = url.pathname.replace(/^\/api\/flows\/?/, "").split("/").filter(Boolean).map(decodeURIComponent); - const method = (init?.method ?? "GET").toUpperCase() as "GET" | "POST" | "PATCH" | "DELETE"; - return router[method](new Request(url, init), { params: { path: segments } }); - }, - }); - await openMailboxes(workforce.mailboxes, { client, userId: "devuser" }); - await flowstate.dispose(); - return workforce.mailboxes.map((mailbox) => mailbox.id).sort(); - } - - it("refuses to boot, names every stale mailbox and the fix, and moves no mailbox", async () => { - const dataDir = await mkdtemp(path.join(tmpdir(), "ks-named-org-")); - cleanups.push(() => rm(dataDir, { recursive: true, force: true })); - const mailboxIds = await preChangeBoot(dataDir); - expect(mailboxIds.length).toBeGreaterThan(0); - - const boot = bootApp({ dataDir }); - await expect(boot).rejects.toThrow(/written before kitchen-sink ran as organization "kitchen-sink"/); - await expect(boot).rejects.toThrow(/Delete the store and restart: .*remove \.fsdev\/data/); - const message = await boot.catch((error: Error) => error.message); - for (const id of mailboxIds) expect(message).toContain(`"${id}" (organization "${DEFAULT_ORG_ID}")`); - - // Nothing was migrated or deleted: every mailbox is still where the old boot left it. - // (The guard only reads. The boot step before it still writes its per-organization - // roster report, as it does on every boot.) - const store = storeAt(dataDir); - for (const id of mailboxIds) expect((await store.session.get(id))?.orgId, id).toBe(DEFAULT_ORG_ID); }); - it("boots over the same location once the store is wiped", async () => { - const dataDir = await mkdtemp(path.join(tmpdir(), "ks-named-org-")); - cleanups.push(() => rm(dataDir, { recursive: true, force: true })); - await preChangeBoot(dataDir); - await rm(path.join(dataDir, ".fsdev", "data"), { recursive: true, force: true }); - - const { router } = await bootApp({ dataDir }); - const help = await call(router, "GET", ["sessions", "support.help"]); - expect(help.status, help.text).toBe(200); - expect(json(help.text).session.orgId).toBe(ORG); - }); -}); - -/** - * A store written before mailboxes were renamed is not carried over either. - * The boot stops before opening anything, names each stale mailbox and what - * gave it away, and says how to reset, as the organization check above does. - * It only reads: the old data is still there afterwards. - * - * Two marks: a session on the old built-in kind, and a mailbox whose kind - * never said anything (a custom kind keeps its name) but whose transcript - * holds a line under the old item name. - */ -describe("a store written before mailboxes were renamed", () => { - const now = Date.now(); - - async function seedSession(dataDir: string, flowKind: string) { - await storeAt(dataDir).session.set( - "support.help", - { - id: "support.help", flowKind, flowId: flowKind, userId: "devuser", orgId: ORG, state: { members: [], instructions: "" }, - lineageId: "lin_support.help", version: 0, createdAt: now, updatedAt: now, journal: [], - } as never, - "absent", - ); - } - - async function seedOldLine(dataDir: string) { - await storeAt(dataDir).request.set( - "req_old_line", - { - id: "req_old_line", flowKind: "mailbox", flowId: "mailbox", actionName: "post", userId: "devuser", sessionId: "support.help", - orgId: ORG, source: "http", status: "completed", startedAtMs: now, state: {}, lineageId: "lin_req_old_line", version: 0, - createdAt: now, updatedAt: now, journal: [], - items: [{ id: "item_old_line", type: "component", component: PRE_RENAME_NAMES.postComponent, data: { body: "hi" }, status: "completed", createdAt: now }], - } as never, - "absent", - ); - } - - async function refusedBoot(dataDir: string) { - const boot = bootApp({ dataDir }); - await expect(boot).rejects.toThrow(/were renamed to mailboxes/); - await expect(boot).rejects.toThrow(/Delete the store and restart: .*remove \.fsdev\/data/); - return boot.catch((error: Error) => error.message); - } - - it("refuses to boot over a session on the old built-in kind, naming the mailbox, and moves nothing", async () => { - const dataDir = await mkdtemp(path.join(tmpdir(), "ks-pre-rename-")); - cleanups.push(() => rm(dataDir, { recursive: true, force: true })); - await seedSession(dataDir, PRE_RENAME_NAMES.kind); - - const message = await refusedBoot(dataDir); - - expect(message).toContain(`mailbox "support.help" (it is a session on the "${PRE_RENAME_NAMES.kind}" kind)`); - expect((await storeAt(dataDir).session.get("support.help"))?.flowKind).toBe(PRE_RENAME_NAMES.kind); + it("binds a specialist's session for the coordinator's post to kitchen-sink too", async () => { + const { router, runtime } = await bootApp(); + const opened = await openSession(router, "coordinator", { orgId: "globex", state: { workerId: "support.help" } }); + expect(opened.status, opened.text).toBe(201); + const posted = await act(router, "coordinator", "run", opened.id!, { message: "a post naming nobody" }, { orgId: "globex" }); + expect(posted.status, posted.text).toBe(200); + // Best fit cannot place it, so the fallback takes it, in a session the post's conversation started. + let delegate: { orgId?: string } | undefined; + for (let i = 0; i < 200 && delegate === undefined; i++) { + const sessions = await runtime.stores.session.list({ flowId: "agent", parentage: "all" }); + delegate = sessions.find((s) => s.parentSessionId === opened.id); + if (delegate === undefined) await new Promise((resolve) => setTimeout(resolve, 25)); + } + expect(delegate, "the fallback's session").toBeDefined(); + expect(delegate!.orgId).toBe(ORG); }); - it("refuses to boot over a mailbox whose transcript holds a line under the old item name", async () => { - const dataDir = await mkdtemp(path.join(tmpdir(), "ks-pre-rename-")); - cleanups.push(() => rm(dataDir, { recursive: true, force: true })); - await seedSession(dataDir, "mailbox"); - await seedOldLine(dataDir); - - const message = await refusedBoot(dataDir); - - expect(message).toContain(`mailbox "support.help" (its transcript holds "${PRE_RENAME_NAMES.postComponent}" items)`); - }); }); diff --git a/apps/kitchen-sink/test/picked-session-panel.test.tsx b/apps/kitchen-sink/test/picked-session-panel.test.tsx index ecf5e05673..b32f2e8252 100644 --- a/apps/kitchen-sink/test/picked-session-panel.test.tsx +++ b/apps/kitchen-sink/test/picked-session-panel.test.tsx @@ -1,21 +1,22 @@ // @vitest-environment happy-dom /** - * The picked session's panel: what a mailbox shows, and what each composer - * sends to which action. + * The picked session's panel: what a coordinator's conversation shows, and + * what each composer sends to which action. * * The page's sends are the whole contract here. A composer that sends the * wrong action, or sends an `author`, would still render; only the recorded * call catches it. The browser half (the line survives a reload, the seat * keeps both sides) is `e2e/talk-from-page.spec.ts` and the goal check. * - * Checks, by the spec's ids (`specs/issues/FIX-1585/PLAN.md` V5). Red states - * produced before these were trusted: - * - label a line `principal ?? author`: the label-order case fails. + * Checks, by the spec's ids (`specs/issues/FIX-1585/PLAN.md` V5), moved from + * the mailbox onto the coordinator by FIX-1792. Red states produced before + * these were trusted: + * - label a line by its role before its `agentName`: the label case fails. * - drop the `text.length === 0` guard on Send: the whitespace case fails. * - clear the draft when the send resolves, not when the stream settles: * the refused-post case fails (the text is gone). - * - send `{ body, author: "devuser" }` from the mailbox composer: the - * mailbox case fails on the recorded input. + * - send `{ message, author: "devuser" }` from the coordinator's composer: + * the coordinator case fails on the recorded input. * - point `agent` at `answer`: the agent case fails. * - settle on the session's `isStreaming` rather than the send's own * request's status: the settles-on-its-own-request case fails (the text @@ -32,7 +33,7 @@ afterEach(cleanup); type Item = Record; -type Run = { id: string; flowId?: string; status?: string }; +type Run = { id: string; flowId?: string; topic?: string; status?: string }; function session( over: { @@ -65,12 +66,22 @@ function session( } as never; } -const line = (id: string, body: string, who: { author?: string; principal?: string } = {}): Item => ({ +/** A kept message: a delegate's answer when `agentName` is given, else a `role` message. */ +const line = (id: string, body: string, who: { agentName?: string; role?: string } = {}): Item => ({ id: `item-${id}`, - type: "component", - component: "mailbox-post", + type: "message", + role: who.role ?? (who.agentName === undefined ? "user" : "assistant"), requestId: `req-${id}`, - data: { id, at: 1, authorVerified: false, body, ...who }, + content: [{ type: "text", text: body }], + ...(who.agentName === undefined ? {} : { agentName: who.agentName }), +}); + +/** A delegate's run, as the coordinator's delivery keys it. */ +const delegateRun = (id: string, worker: string, status?: string): Run => ({ + id, + flowId: "agent", + topic: `delegate:${JSON.stringify([worker, null])}`, + ...(status === undefined ? {} : { status }), }); /** The server's answer for each request id; anything unlisted is still running. */ @@ -83,7 +94,7 @@ afterEach(() => { function panel(kind: string, s: never) { return render( - } />, + } />, ); } @@ -91,49 +102,50 @@ async function type(label: string, text: string) { fireEvent.change(screen.getByLabelText(label), { target: { value: text } }); } -describe("a mailbox's panel", () => { - it("shows the transcript oldest first, each line under its author, else its principal, else unattributed", () => { +describe("a coordinator's panel", () => { + it("shows the lines oldest first: a delegate's answer under its name, the person's post under theirs, the coordinator's own word under it", () => { panel( - "mailbox", + "coordinator", session({ items: [ - line("1", "from a seat", { author: "support.devices", principal: "devuser" }), - { id: "noise", type: "message", role: "assistant", content: [] }, - line("2", "from the page", { principal: "devuser" }), - line("3", "from the digest kind"), + line("1", "from the page"), + { id: "record", type: "component", component: "coordinator-route", data: { by: "evaluated" } }, + line("2", "from a delegate", { agentName: "support.devices" }), + { id: "noise", type: "message", role: "assistant", transient: true, content: [{ type: "text", text: "thinking" }] }, + line("3", "Nobody took this post: no delegate can be reached.", { role: "assistant" }), ], }), ); - const labels = screen.getAllByTestId("mailbox-line-label").map((el) => el.textContent); - const bodies = screen.getAllByTestId("mailbox-line-body").map((el) => el.textContent); - expect(bodies).toEqual(["from a seat", "from the page", "from the digest kind"]); - expect(labels).toEqual(["support.devices", "devuser", "unattributed"]); - // The stream is not drawn for a mailbox: its transcript is. + const labels = screen.getAllByTestId("coordinator-line-label").map((el) => el.textContent); + const bodies = screen.getAllByTestId("coordinator-line-body").map((el) => el.textContent); + expect(bodies).toEqual(["from the page", "from a delegate", "Nobody took this post: no delegate can be reached."]); + expect(labels).toEqual(["devuser", "support.devices", "coordinator"]); + // The stream is not drawn for a coordinator: its lines are. expect(screen.queryByTestId("stream-items")).toBeNull(); }); - it("shows an empty transcript and still posts, when the session holds no mailbox-post item", async () => { + it("shows no lines and still posts, when the session holds no kept message", async () => { const sendAction = vi.fn(async () => ({ status: "in_progress", request: { id: "req-1" } })); - panel("mailbox", session({ items: [{ id: "x", type: "message", role: "assistant", content: [] }], sendAction })); - expect(screen.queryAllByTestId("mailbox-line")).toHaveLength(0); - await type("Post to this mailbox", "hello"); + panel("coordinator", session({ items: [{ id: "x", type: "component", component: "coordinator-route", data: {} }], sendAction })); + expect(screen.queryAllByTestId("coordinator-line")).toHaveLength(0); + await type("Post to this coordinator", "hello"); fireEvent.click(screen.getByRole("button", { name: "Send" })); await waitFor(() => expect(sendAction).toHaveBeenCalledTimes(1)); }); - it("posts the body alone to the mailbox's own post action", async () => { + it("posts the message alone to the coordinator's door", async () => { const sendAction = vi.fn(async () => ({ status: "in_progress", request: { id: "req-1" } })); - panel("mailbox", session({ sendAction })); - await type("Post to this mailbox", " a line "); + panel("coordinator", session({ sendAction })); + await type("Post to this coordinator", " a line "); fireEvent.click(screen.getByRole("button", { name: "Send" })); await waitFor(() => expect(sendAction).toHaveBeenCalledTimes(1)); - expect(sendAction.mock.calls[0]).toEqual(["post", { body: "a line" }]); + expect(sendAction.mock.calls[0]).toEqual(["run", { message: "a line" }]); }); it("will not send whitespace", async () => { const sendAction = vi.fn(); - panel("mailbox", session({ sendAction })); - await type("Post to this mailbox", " \n "); + panel("coordinator", session({ sendAction })); + await type("Post to this coordinator", " \n "); const send = screen.getByRole("button", { name: "Send" }) as HTMLButtonElement; expect(send.disabled).toBe(true); fireEvent.submit(screen.getByTestId("picked-composer")); @@ -142,8 +154,8 @@ describe("a mailbox's panel", () => { it("shows a refused post's reason and keeps the text, then clears it once a post is kept", async () => { const sendAction = vi.fn(async () => ({ status: "in_progress", request: { id: "req-refused" } })); - const view = panel("mailbox", session({ sendAction, isStreaming: true })); - await type("Post to this mailbox", "please keep me"); + const view = panel("coordinator", session({ sendAction, isStreaming: true })); + await type("Post to this coordinator", "please keep me"); fireEvent.click(screen.getByRole("button", { name: "Send" })); await waitFor(() => expect(sendAction).toHaveBeenCalledTimes(1)); @@ -152,22 +164,23 @@ describe("a mailbox's panel", () => { , ); - expect((await screen.findByRole("alert")).textContent).toContain("mailbox-not-bound"); - expect((screen.getByLabelText("Post to this mailbox") as HTMLTextAreaElement).value).toBe("please keep me"); + expect((await screen.findByRole("alert")).textContent).toContain("No post is being routed"); + expect((screen.getByLabelText("Post to this coordinator") as HTMLTextAreaElement).value).toBe("please keep me"); expect((screen.getByRole("button", { name: "Send" }) as HTMLButtonElement).disabled).toBe(false); // A send the door refuses outright says why, and keeps the text too. sendAction.mockRejectedValueOnce(new Error("Request failed (409)")); fireEvent.click(screen.getByRole("button", { name: "Send" })); expect((await screen.findByRole("alert")).textContent).toContain("409"); - expect((screen.getByLabelText("Post to this mailbox") as HTMLTextAreaElement).value).toBe("please keep me"); + expect((screen.getByLabelText("Post to this coordinator") as HTMLTextAreaElement).value).toBe("please keep me"); // A post the server keeps clears the composer. statuses["req-kept"] = "completed"; @@ -178,15 +191,16 @@ describe("a mailbox's panel", () => { , ); await waitFor(() => - expect((screen.getByLabelText("Post to this mailbox") as HTMLTextAreaElement).value).toBe(""), + expect((screen.getByLabelText("Post to this coordinator") as HTMLTextAreaElement).value).toBe(""), ); expect(screen.queryByRole("alert")).toBeNull(); }); @@ -195,8 +209,8 @@ describe("a mailbox's panel", () => { describe("a send settles on its own request", () => { it("keeps the text while another request ends, and shows the failure when its own does", async () => { const sendAction = vi.fn(async () => ({ status: "in_progress", request: { id: "req-mine" } })); - const view = panel("mailbox", session({ sendAction, isStreaming: true })); - await type("Post to this mailbox", "not yet kept"); + const view = panel("coordinator", session({ sendAction, isStreaming: true })); + await type("Post to this coordinator", "not yet kept"); fireEvent.click(screen.getByRole("button", { name: "Send" })); await waitFor(() => expect(sendAction).toHaveBeenCalledTimes(1)); @@ -205,13 +219,14 @@ describe("a send settles on its own request", () => { view.rerender( , ); // This send is still running, so nothing settles. await new Promise((resolve) => setTimeout(resolve, 50)); - expect((screen.getByLabelText("Post to this mailbox") as HTMLTextAreaElement).value).toBe("not yet kept"); + expect((screen.getByLabelText("Post to this coordinator") as HTMLTextAreaElement).value).toBe("not yet kept"); expect(screen.queryByRole("alert")).toBeNull(); // It asked the server about this send, not about the session. expect(requestStatus).toHaveBeenCalledWith("req-mine"); @@ -220,7 +235,7 @@ describe("a send settles on its own request", () => { // shows once the next check sees it, and the text stays. statuses["req-mine"] = "failed"; expect((await screen.findByRole("alert", undefined, { timeout: 2000 })).textContent).toContain("failed"); - expect((screen.getByLabelText("Post to this mailbox") as HTMLTextAreaElement).value).toBe("not yet kept"); + expect((screen.getByLabelText("Post to this coordinator") as HTMLTextAreaElement).value).toBe("not yet kept"); }); }); @@ -230,7 +245,7 @@ describe("switching picks", () => { const view = panel("agent", session({ sessionId: "sess-a", sendAction })); await type("Message this seat", "meant for the first seat"); - view.rerender(); + view.rerender(); expect((screen.getByLabelText("Message this seat") as HTMLTextAreaElement).value).toBe(""); expect((screen.getByRole("button", { name: "Send" }) as HTMLButtonElement).disabled).toBe(true); }); @@ -265,59 +280,59 @@ describe("a seat's panel", () => { describe("who is working", () => { const rows = () => screen.queryAllByTestId("working-row").map((el) => el.textContent); const rerender = (view: ReturnType, kind: string, s: never) => - view.rerender(); + view.rerender(); - it.each(["mailbox", "agent"])("a %s panel names each unfinished run by its flow", (kind) => { - panel(kind, session({ childSessions: [{ id: "run-1", flowId: "support.devices", status: "active" }] })); - expect(rows()).toEqual(["support.devices is working"]); + it.each(["coordinator", "agent"])("a %s panel names each unfinished run by the delegate it was handed to, else its flow", (kind) => { + panel(kind, session({ childSessions: [delegateRun("run-1", "support.devices", "active"), { id: "run-2", flowId: "agent", status: "active" }] })); + expect(rows()).toEqual(["support.devices is working", "agent is working"]); }); it("clears the row when the run finishes, however it ends, and keeps the line it posted", () => { - const working = [{ id: "run-1", flowId: "support.devices", status: "active" }]; - const view = panel("mailbox", session({ childSessions: working })); + const working = [delegateRun("run-1", "support.devices", "active")]; + const view = panel("coordinator", session({ childSessions: working })); expect(rows()).toEqual(["support.devices is working"]); for (const status of ["completed", "failed", "incomplete", "aborted"]) { - rerender(view, "mailbox", session({ childSessions: working })); + rerender(view, "coordinator", session({ childSessions: working })); expect(rows()).toEqual(["support.devices is working"]); rerender( view, - "mailbox", + "coordinator", session({ - childSessions: [{ id: "run-1", flowId: "support.devices", status }], - items: [line("o", "refunds post on Fridays", { author: "support.devices" })], + childSessions: [delegateRun("run-1", "support.devices", status)], + items: [line("o", "refunds post on Fridays", { agentName: "support.devices" })], }), ); expect(rows()).toEqual([]); - expect(screen.getAllByTestId("mailbox-line-body").map((el) => el.textContent)).toEqual([ + expect(screen.getAllByTestId("coordinator-line-body").map((el) => el.textContent)).toEqual([ "refunds post on Fridays", ]); } }); it("reads a run with no status, which has no run to speak of, as not working", () => { - panel("mailbox", session({ childSessions: [{ id: "run-1", flowId: "support.devices" }] })); + panel("coordinator", session({ childSessions: [delegateRun("run-1", "support.devices")] })); expect(rows()).toEqual([]); }); - it("shows two seats working on one post as two rows, each clearing on its own", () => { + it("shows two delegates working on one post as two rows, each clearing on its own", () => { const view = panel( - "mailbox", + "coordinator", session({ childSessions: [ - { id: "run-1", flowId: "support.devices", status: "active" }, - { id: "run-2", flowId: "support.accounts", status: "active" }, + delegateRun("run-1", "support.devices", "active"), + delegateRun("run-2", "support.accounts", "active"), ], }), ); expect(rows()).toEqual(["support.devices is working", "support.accounts is working"]); rerender( view, - "mailbox", + "coordinator", session({ childSessions: [ - { id: "run-1", flowId: "support.devices", status: "completed" }, - { id: "run-2", flowId: "support.accounts", status: "active" }, + delegateRun("run-1", "support.devices", "completed"), + delegateRun("run-2", "support.accounts", "active"), ], }), ); @@ -325,7 +340,7 @@ describe("who is working", () => { }); it("shows a run with no recorded flow as background work, not a seat", () => { - panel("mailbox", session({ childSessions: [{ id: "run-1", status: "active" }] })); + panel("coordinator", session({ childSessions: [{ id: "run-1", status: "active" }] })); expect(rows()).toEqual(["Background work is running"]); }); @@ -333,35 +348,35 @@ describe("who is working", () => { // finished since. The row says when it was true, not that it is true now. it("says a row was true at the last check while the list could not be read again", () => { const working = [ - { id: "run-1", flowId: "support.devices", status: "active" }, + delegateRun("run-1", "support.devices", "active"), { id: "run-2", status: "active" }, ]; - const view = panel("mailbox", session({ childSessions: working, childSessionsStale: true })); + const view = panel("coordinator", session({ childSessions: working, childSessionsStale: true })); expect(rows()).toEqual([ "support.devices was working at the last check", "Background work was running at the last check", ]); - rerender(view, "mailbox", session({ childSessions: working })); + rerender(view, "coordinator", session({ childSessions: working })); expect(rows()).toEqual(["support.devices is working", "Background work is running"]); }); it("keeps no timer of its own, and keeps the draft, as lines land and runs end", async () => { vi.useFakeTimers(); try { - const view = panel("mailbox", session({ childSessions: [{ id: "run-1", flowId: "support.devices", status: "active" }] })); - fireEvent.change(screen.getByLabelText("Post to this mailbox"), { target: { value: "half a thought" } }); + const view = panel("coordinator", session({ childSessions: [delegateRun("run-1", "support.devices", "active")] })); + fireEvent.change(screen.getByLabelText("Post to this coordinator"), { target: { value: "half a thought" } }); rerender( view, - "mailbox", + "coordinator", session({ - childSessions: [{ id: "run-1", flowId: "support.devices", status: "completed" }], - items: [line("o", "refunds post on Fridays", { author: "support.devices" })], + childSessions: [delegateRun("run-1", "support.devices", "completed")], + items: [line("o", "refunds post on Fridays", { agentName: "support.devices" })], }), ); expect(vi.getTimerCount()).toBe(0); // Not remounted: the draft is still there. - expect((screen.getByLabelText("Post to this mailbox") as HTMLTextAreaElement).value).toBe("half a thought"); + expect((screen.getByLabelText("Post to this coordinator") as HTMLTextAreaElement).value).toBe("half a thought"); } finally { vi.useRealTimers(); } diff --git a/apps/kitchen-sink/test/shell-imports.test.ts b/apps/kitchen-sink/test/shell-imports.test.ts index bdc2632933..3e74294d30 100644 --- a/apps/kitchen-sink/test/shell-imports.test.ts +++ b/apps/kitchen-sink/test/shell-imports.test.ts @@ -9,6 +9,10 @@ * Red state: on the tree before the shell was rebuilt, the page imports * `components/session-sidebar.tsx`, which types and draws a session list of * its own. The first and last cases both fail there. + * + * The panel drew the package's `BoardList` for `support.help`'s escalations + * board until the escalation feature was removed (FIX-1792): no board is left + * for it to draw, so it is the roster alone. */ import { existsSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; @@ -43,12 +47,8 @@ describe("V9 · the shell's rail and panel are package components", () => { expect(page).not.toMatch(/components\/session-sidebar/); }); - it("draws the panel with the package's board list, and the roster from Workforce's client", () => { + it("draws the panel's roster from Workforce's client", () => { const panel = read("components/team-panel.tsx"); - const fromReact = importedFrom(panel, "@flow-state-dev/react"); - expect(fromReact).toEqual(expect.arrayContaining(["BoardList"])); - expect(fromReact).not.toContain("BoardColumns"); - expect(panel).toContain(" { - it("runs the four specialists as standard workers on one copy of agent, and registers no copy per worker", async () => { +describe("V1 · the boot serves one best-fit coordinator and four specialists", () => { + it("runs the specialists on one copy of agent and support.help on one copy of coordinator, and no copy per worker", async () => { const { get, workforce } = await boot(); const roster = await workforce.roster(); expect(roster.map((worker) => [worker.id, worker.flow, worker.standard]).sort()).toEqual([ @@ -76,69 +81,58 @@ describe("V1 · the boot serves one routed mailbox and four specialists", () => ["support.devices", "agent", true], ["support.fsd", "agent", true], ["support.general", "agent", true], + ["support.help", "coordinator", true], ]); const { flows } = (await get([])) as { flows: Array<{ id: string; kind: string }> }; expect(flows.filter((flow) => flow.kind === "agent").map((flow) => flow.id)).toEqual(["agent"]); + expect(flows.filter((flow) => flow.kind === "coordinator").map((flow) => flow.id)).toEqual(["coordinator"]); expect(flows.filter((flow) => roster.some((worker) => worker.id === flow.id))).toEqual([]); }); - it("opens one mailbox, support.help, on the built-in mailbox kind, beside the app's own flows", async () => { + it("serves the coordinator flow beside the app's own flows, and no mailbox flow", async () => { const { get } = await boot(); const { flows } = (await get([])) as { flows: Array<{ id: string; kind: string }> }; - // The workforce's kinds are the built-ins: `agent`, the mailbox and the roster flow. The rest are the app's own flows. + // The workforce's kinds are the built-ins: `agent`, `coordinator` and the roster flow. The rest are the app's own flows. expect([...new Set(flows.map((flow) => flow.kind))].sort()).toEqual([ "agent", "chat-agent", - "mailbox", + "coordinator", "rich-text-component", "weekly-digest", "workforce-roster", ]); - const { sessions } = (await get(["sessions"], "?flowId=mailbox&limit=100")) as { sessions: Array<{ id: string }> }; - expect(sessions.map((session) => session.id)).toEqual(["support.help"]); }); - it("lists support.help alone in the rail, though a kept store still holds a retired mailbox", async () => { - const { sessions } = await boot(); + it("lists support.help in the rail as the person's one conversation with it, a session on coordinator naming it", async () => { + const { sessions, workforce } = await boot(); const { railSessions } = await import("@/lib/rail-sessions"); - // What an earlier roster leaves in a store kept across the upgrade: a - // session of the same mailbox kind, for a mailbox the tree no longer declares. - await sessions.createSession({ flowKind: "mailbox", userId: "devuser", sessionId: "support.desk" }); - // Not vacuous: the store holds it, and a listing by kind returns it. - const stored = await sessions.listSessions({ flowKind: "mailbox", userId: "devuser" }); - expect(stored.map((session) => session.id).sort()).toEqual(["support.desk", "support.help"]); + // Not vacuous: on a fresh store the coordinator kind's own listing holds nothing. + expect(await sessions.listSessions({ flowKind: "coordinator", userId: "devuser" })).toEqual([]); - // The navigator's own query for a singleton kind's leaf. - const listed = await railSessions(sessions).listSessions({ - flowKind: "mailbox", - userId: "devuser", - include: "dispatch-runs", - }); - expect(listed.map((session) => session.id)).toEqual(["support.help"]); + // The navigator's own query for the kind's leaf. + const query = { flowKind: "coordinator", userId: "devuser", include: "dispatch-runs" } as const; + const listed = await railSessions(sessions, workforce).listSessions(query); + expect(listed.map((session) => [session.title, session.flowKind])).toEqual([["support.help", "coordinator"]]); + // The same conversation every time, and the one the coordinator's id finds (BR-7a). + const again = await railSessions(sessions, workforce).listSessions(query); + expect(again.map((session) => session.id)).toEqual(listed.map((session) => session.id)); + expect((await workforce.findWorkerSession({ worker: "support.help" }))?.id).toBe(listed[0]!.id); }); it("lists every other kind in the rail as the store has it", async () => { - const { sessions } = await boot(); + const { sessions, workforce } = await boot(); const { railSessions } = await import("@/lib/rail-sessions"); const created = await sessions.createSession({ flowKind: "chat-agent", userId: "devuser" }); const query = { flowKind: "chat-agent", userId: "devuser" } as const; - const listed = await railSessions(sessions).listSessions(query); + const listed = await railSessions(sessions, workforce).listSessions(query); expect(listed.map((session) => session.id)).toContain(created.id); expect(listed).toEqual(await sessions.listSessions(query)); }); - it("warns once that a board is unattended, and names support.help's escalations", async () => { - const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); - await boot(); - const unattended = warn.mock.calls.map((call) => call.join(" ")).filter((line) => line.includes("holds board")); - expect(unattended).toHaveLength(1); - expect(unattended[0]).toContain('mailbox "support.help" holds board "escalations"'); - }); - - it("generates escalate into the catalog, and no kind of the app's own", async () => { + it("generates no block and no kind of the app's own", async () => { const generated = await import("@/workforce/workforce.gen"); expect(Object.keys(generated.kinds)).toEqual([]); expect(Object.keys(generated.mailboxKinds)).toEqual([]); - expect(Object.keys(generated.blocks)).toEqual(["escalate"]); + expect(Object.keys(generated.blocks)).toEqual([]); }); }); diff --git a/apps/kitchen-sink/test/workforce-shell.test.ts b/apps/kitchen-sink/test/workforce-shell.test.ts index 35d73f5475..45b75d2305 100644 --- a/apps/kitchen-sink/test/workforce-shell.test.ts +++ b/apps/kitchen-sink/test/workforce-shell.test.ts @@ -1,74 +1,53 @@ /** * The names the shell writes down, held to the tree they describe. * - * The rail's kind lists and the panel's board ids are literals in + * The rail's kind lists and the coordinators it lists are literals in * `lib/workforce-shell.ts`, because a browser cannot read `workforce/`. That - * makes them a second copy, and a copy drifts: add a worker flow or a board to - * a `MAILBOX.md`, and the rail or the panel silently shows the old one. These + * makes them a second copy, and a copy drifts: add a worker flow or a + * coordinator's `WORKER.md`, and the rail silently shows the old one. These * cases fail on that drift. * * Red states produced before these were trusted: - * - add a kind to the tree and not to `SEAT_KINDS`: the seat-kind case fails. + * - add a kind to the tree and not to `SEAT_KINDS` or `COORDINATOR_KINDS`: + * the worker-flow case fails. * - drop `agent`'s entry from `SEAT_ASKS`: the answering-action case fails, * naming the kind with no entry. * - misname `agent`'s field `message`: the answering-action case fails, * naming the field the kind actually takes. - * - drop the board from `SHELL_BOARDS`: the board case fails. - * - drop the board from `support.help`'s `MAILBOX.md`: the board case and - * the mailbox flow's declaration case both fail. + * - drop `support.help` from `SHELL_COORDINATORS`: the coordinator case + * fails. */ import { describe, expect, it } from "vitest"; import { z, type ZodTypeAny } from "zod"; -import { MAILBOX_KIND, mailboxBoard, mailboxBoardIds } from "@flow-state-dev/workforce"; -import { readMailboxesDirectory, readWorkforce } from "@flow-state-dev/workforce/loader"; +import { COORDINATOR_KIND } from "@flow-state-dev/workforce"; +import { readWorkforce } from "@flow-state-dev/workforce/loader"; import { buildKitchenSinkWorkforce, workforceRoot } from "../workforce/hire"; -import { mailboxKinds } from "../workforce/workforce.gen"; import { - MAILBOX_KINDS, + COORDINATOR_ASK, + COORDINATOR_KINDS, defaultConversation, + delegateOfRun, SEAT_ASKS, SEAT_KINDS, - SHELL_BOARDS, - SHELL_MAILBOXES, + SHELL_COORDINATORS, seatAskFor, - wokenWorkerOf, } from "../lib/workforce-shell"; describe("the shell's names match the workforce tree", () => { - it("lists every worker flow the app runs workers on", async () => { + it("lists every worker flow the app runs workers on, the coordinator's apart", async () => { const { installation } = await buildKitchenSinkWorkforce(); - expect([...SEAT_KINDS].sort()).toEqual(Object.keys(installation.workerFlows()).sort()); + expect([...SEAT_KINDS, ...COORDINATOR_KINDS].sort()).toEqual(Object.keys(installation.workerFlows()).sort()); + expect([...COORDINATOR_KINDS]).toEqual([COORDINATOR_KIND]); }); - it("lists every mailbox kind: the framework's own, plus the tree's", () => { - expect([...MAILBOX_KINDS].sort()).toEqual(["mailbox", ...Object.keys(mailboxKinds)].sort()); - }); - - it("draws every board the tree's mailboxes declare, under the id the package mints", async () => { - const { mailboxes, errors } = await readMailboxesDirectory(workforceRoot); + it("lists every coordinator the tree declares, by its worker id", async () => { + const { workers, errors } = await readWorkforce(workforceRoot); expect(errors).toEqual([]); - const declared = mailboxBoardIds(mailboxes); - // Not vacuous: the tree does declare boards. + const declared = workers.filter((worker) => worker.declared.flow === COORDINATOR_KIND).map((worker) => worker.id); + // Not vacuous: the tree does declare a coordinator. expect(declared.length).toBeGreaterThan(0); - expect(SHELL_BOARDS.map((board) => board.ref).sort()).toEqual(declared); - for (const board of SHELL_BOARDS) { - expect(mailboxBoard(board.mailboxId, board.board).id).toBe(board.ref); - } - }); - - it("lists every mailbox the tree declares, under the kind its file selects", async () => { - const { mailboxes, errors } = await readMailboxesDirectory(workforceRoot); - expect(errors).toEqual([]); - // Not vacuous: the tree does declare mailboxes. - expect(mailboxes.length).toBeGreaterThan(0); - // A file with no `flow:` line selects the built-in kind. - const declared = mailboxes.map((mailbox) => ({ - id: mailbox.id, - kind: (mailbox.declared as { flow?: string }).flow ?? MAILBOX_KIND, - })); - const byId = (a: { id: string }, b: { id: string }) => a.id.localeCompare(b.id); - expect(SHELL_MAILBOXES.map(({ id, kind }) => ({ id, kind })).sort(byId)).toEqual(declared.sort(byId)); + expect([...SHELL_COORDINATORS].sort()).toEqual(declared.sort()); }); }); @@ -111,36 +90,10 @@ describe("each seat kind's composer sends to an action the kind declares", () => }); }); -/** Whether `resources` declares `ref` so a browser may read its rows. */ -function expectBrowserReadable(resources: Record, ref: string, flow: string): void { - const declared = resources[ref] as { client?: { state?: { read?: boolean } } } | undefined; - expect(declared, `${flow} declares no resource "${ref}"`).toBeDefined(); - expect(declared!.client?.state?.read).toBe(true); -} - -describe("each board the panel reads is declared where it reads it", () => { - // Each board is read through its mailbox's session, so it is the flow that - // session runs on, the one the mailbox's file selects, that must declare it. - it.each(SHELL_BOARDS.map((board) => [board.ref, board] as const))( - "board %s, readable by a browser through its mailbox's session", - async (_ref, board) => { - const { mailboxFlows } = await buildKitchenSinkWorkforce(); - const mailbox = SHELL_MAILBOXES.find(({ id }) => id === board.mailboxId); - expect(mailbox, `SHELL_MAILBOXES has no mailbox "${board.mailboxId}"`).toBeDefined(); - const flow = mailboxFlows.find(({ kind }) => kind === mailbox!.kind); - expect(flow, `no mailbox flow of kind "${mailbox!.kind}"`).toBeDefined(); - expectBrowserReadable( - (flow!.resources ?? {}) as Record, - board.ref, - `the "${mailbox!.kind}" mailbox flow`, - ); - }, - ); - - it("uses refs the collection route can address: one path segment each", () => { - for (const ref of SHELL_BOARDS.map((b) => b.ref)) { - expect(ref).not.toContain("/"); - } +describe("a coordinator's composer sends to the coordinator flow's door", () => { + it("names the action that takes exactly the post, one string field", async () => { + const actions = await oneStringActions(COORDINATOR_KIND); + expect(actions[COORDINATOR_ASK.action], `the coordinator's one-string actions: ${JSON.stringify(actions)}`).toBe(COORDINATOR_ASK.field); }); }); @@ -161,12 +114,14 @@ describe("the conversation the page opens on", () => { }); }); -describe("a mailbox's working row names the worker it woke", () => { +describe("a coordinator's working row names the delegate it handed the post to", () => { // Every worker runs on the one `agent` copy, so a run's flow id is `agent` - // for all of them; only the wake's key says which worker it is. - it("reads the worker off the wake's key, and nothing off any other run", () => { - expect(wokenWorkerOf({ topic: "mailbox:support.help:support.devices" })).toBe("support.devices"); - expect(wokenWorkerOf({ topic: "brief:billing" })).toBeUndefined(); - expect(wokenWorkerOf({})).toBeUndefined(); + // for all of them; only the delivery's key says which worker it is. The key + // a real delivery carries is held in `test/mailbox-wake.test.ts`. + it("reads the worker off the delivery's key, and nothing off any other run", () => { + expect(delegateOfRun({ topic: `delegate:${JSON.stringify(["support.devices", null])}` })).toBe("support.devices"); + expect(delegateOfRun({ topic: "delegate:not json" })).toBeUndefined(); + expect(delegateOfRun({ topic: "brief:billing" })).toBeUndefined(); + expect(delegateOfRun({})).toBeUndefined(); }); }); diff --git a/apps/kitchen-sink/workforce/blocks/escalate.ts b/apps/kitchen-sink/workforce/blocks/escalate.ts deleted file mode 100644 index 508d3b1cad..0000000000 --- a/apps/kitchen-sink/workforce/blocks/escalate.ts +++ /dev/null @@ -1,138 +0,0 @@ -/** - * `escalate`: a specialist files a case that needs a person onto - * `support.help`'s `escalations` board, through that mailbox's own `fileTask`. - * - * Declared by living in `workforce/blocks/`: `fsdev gen` puts it on the - * generated `blocks` map under its basename, and the `agent` kind carries that - * map as its tool catalog. A seat reaches it only by naming `escalate` in its - * `tools:`, as each specialist's `WORKER.md` does. - * - * The model writes only the case. The mailbox and the board are named here, - * and the author is the worker the turn loaded, which the server checked when - * the conversation was created, so no model can file as another worker or - * onto another board. - * - * The dispatch is fire-and-forget: the row is written when the mailbox runs - * `fileTask`, a moment after the tool returns, so the tool never sees the - * mailbox's answer. The one refusal it could predict, it checks first: the - * mailbox files only its members' cases, so a worker that is not one (a user - * can hire a worker of their own with this tool) gets `filed: false` and sends - * nothing, rather than a `filed: true` the mailbox would then refuse. Nothing - * drains `escalations` in this app, and the boot says so. - */ -import { DispatchRefusedError, dispatcher, handler, sequencer, type BlockContext } from "@flow-state-dev/core"; -import { MAILBOX_KIND, verifiedWorkerOf } from "@flow-state-dev/workforce"; -import { z } from "zod"; - -/** The mailbox a case is filed in. */ -export const ESCALATION_MAILBOX = "support.help"; - -/** The board on that mailbox a case is filed onto. */ -export const ESCALATION_BOARD = "escalations"; - -/** - * The seats the mailbox files for: its `members:`, as `support.help`'s - * `MAILBOX.md` declares them. Written down because the mailbox's own list lives - * in its session, which a seat's tool cannot read; `test/escalate.test.ts` - * holds this to the file. - */ -export const ESCALATION_MEMBERS: readonly string[] = [ - "support.devices", - "support.accounts", - "support.fsd", - "support.general", -]; - -/** The tool's input: the case, and only the case. */ -export const escalateInput = z.object({ - case: z.string().min(1).describe("What the person needs, in a sentence or two, for whoever picks it up."), -}); - -/** - * What the model reads about the tool. Shared with the goal check's stand-in - * (`lib/escalate-control.ts`), so the model is offered the same tool either way. - */ -export const ESCALATE_DESCRIPTION = - "File this case for a person to pick up, when you cannot resolve it yourself. " + - "Say in your answer that you filed it."; - -/** The dispatch refusal an error carries, directly or as its cause. */ -function refusalIn(error: unknown): DispatchRefusedError | undefined { - for (let current = error; current instanceof Error; current = current.cause) { - if (current instanceof DispatchRefusedError) return current; - } - return undefined; -} - -/** - * What the tool hands the model when the deployment cannot deliver into the - * mailbox's session: a dispatcher that hands work to an external queue refuses - * a delivery into an existing session, so there is nowhere to file. - */ -const filingUnavailable = handler({ - name: "escalate-unavailable", - // Whatever the dispatch threw: narrowed below, and rethrown unless it is the refusal. - inputSchema: z.unknown(), - outputSchema: z.object({ filed: z.literal(false), reason: z.string() }), - execute: (error: unknown) => { - // Only the external-dispatcher refusal is this deployment's to report. Any - // other failure stays one. - if (refusalIn(error)?.refused !== "external-dispatcher") throw error; - return { - filed: false as const, - reason: - "Filing is unavailable here: this deployment runs work on an external queue, which " + - "cannot deliver into the support mailbox. Nothing was filed; tell the person so.", - }; - }, -}); - -/** The dispatch into the mailbox's own `fileTask`, signed as the seat. */ -const fileIntoMailbox = dispatcher({ - name: "escalate-file", - flowKind: MAILBOX_KIND, - action: "fileTask", - inputSchema: escalateInput, - session: { id: () => ESCALATION_MAILBOX }, - payload: (input, ctx: BlockContext) => ({ - board: ESCALATION_BOARD, - goal: input.case, - author: seatIdOf(ctx), - }), -}); - -/** The calling worker's id: the one this turn loaded and checked. */ -function seatIdOf(ctx: BlockContext): string | undefined { - return verifiedWorkerOf(ctx.session as object); -} - -/** Whether the calling seat is one the mailbox files for. */ -const isMember = (ctx: BlockContext): boolean => ESCALATION_MEMBERS.includes(seatIdOf(ctx) ?? ""); - -/** A member's case: into the mailbox's `fileTask`. */ -const fileAsMember = sequencer({ name: "escalate-member", inputSchema: escalateInput }) - .step(fileIntoMailbox) - .map(() => ({ filed: true as const })) - .rescue([{ block: filingUnavailable }]); - -/** Any other seat's: nothing is sent, and the model is told why. */ -const notAMember = handler({ - name: "escalate-not-a-member", - inputSchema: escalateInput, - outputSchema: z.object({ filed: z.literal(false), reason: z.string() }), - execute: () => ({ - filed: false as const, - reason: - `Only the members of ${ESCALATION_MAILBOX} can file on its ${ESCALATION_BOARD} board, and this ` + - "seat is not one. Nothing was filed; tell the person so.", - }), -}); - -export default sequencer({ - name: "escalate", - description: ESCALATE_DESCRIPTION, - inputSchema: escalateInput, -}).branch({ - member: [(input: z.infer) => input, (_input, ctx) => isMember(ctx as BlockContext), fileAsMember], - outsider: [(input: z.infer) => input, (_input, ctx) => !isMember(ctx as BlockContext), notAMember], -}); diff --git a/apps/kitchen-sink/workforce/hire.ts b/apps/kitchen-sink/workforce/hire.ts index 35fe28d6e3..1ee1d42de3 100644 --- a/apps/kitchen-sink/workforce/hire.ts +++ b/apps/kitchen-sink/workforce/hire.ts @@ -4,13 +4,13 @@ * The point of this module is what it does *not* contain: no worker is named * here, and no block is. `blocks` comes from `workforce.gen.ts`, which * `fsdev gen` writes from the tree beside this file, and the workers come from - * the `teams/` folders: one mailbox, `support.help`, and four specialists on - * the built-in `agent` flow. Adding a tool means adding a file and re-running - * the command — there is no second place to edit. + * the `teams/` folders: one coordinator, `support.help`, and four specialists + * on the built-in `agent` flow. Adding a tool means adding a file and + * re-running the command — there is no second place to edit. * - * The specialists are standard workers: every user has them, and they run on - * one registered copy of `agent`. A user's own workers, hired or forked - * through the roster flow, run on that same copy. + * Every worker is a standard worker: every user has it, and it runs on one + * registered copy of its flow. A user's own workers, hired or forked through + * the roster flow, run on the same copies. * * This subtree is the whole of the app's Layer 2 usage. Nothing in `app/` * reaches into it. One edge leads in: `fsdev.config.ts`, which awaits the @@ -20,72 +20,38 @@ * generated module of static imports survives a production build) was asserted * and untested (FIX-1429). * - * The mailbox is routed: each post from a person reaches the one specialist - * whose `description:` fits it (`routeByPurpose`, on `ROUTE_MODEL`), and that - * specialist's answer lands in the mailbox under its name. + * `support.help` is a coordinator on `routing: best-fit`: each post from a + * person reaches the one specialist whose `description:` fits it (one + * evaluator call on `ROUTE_MODEL`), and that specialist's answer lands in the + * person's conversation with the coordinator under its name. */ import { + COORDINATOR_KIND, createWorkerInstallation, defineAgentWorkerFlow, - defineMailboxFlow, + defineCoordinatorFlow, hireWorkforce, - mailboxBoardIds, - mailboxInstances, - routeByPurpose, - workerMailboxPostCapability, - type MailboxManifest, type WorkerInstallation, } from "@flow-state-dev/workforce"; -import { readMailboxesDirectory, readWorkforce } from "@flow-state-dev/workforce/loader"; +import { readWorkforce } from "@flow-state-dev/workforce/loader"; import type { FlowInstance } from "@flow-state-dev/core/types"; import { fileURLToPath } from "node:url"; import { dirname } from "node:path"; -import { mailboxLandingControl } from "../lib/mailbox-landing-control"; -import { mailboxPostControl } from "../lib/mailbox-post-control"; -import { withMailboxRouteControl } from "../lib/mailbox-route-control"; -import { escalateControl } from "../lib/escalate-control"; +import { coordinatorWorkersUnderControl, delegateFlowsUnderControl } from "../lib/coordinator-control"; +import { landingControl } from "../lib/landing-control"; import { ROUTE_MODEL } from "../lib/models"; -import { notifyFor } from "./mailbox-notify"; -import { blocks, mailboxKinds, kinds, packageBlocks, seatBlocks } from "./workforce.gen"; - -/** - * `post-to-mailbox`, so a specialist can answer in a mailbox it belongs to, - * under its own name. Offered, not granted: only a worker whose file names the - * tool can post, and every specialist's does. The goal check's - * `post-without-author` control swaps in a stand-in, in test mode only. - */ -const mailboxPost = mailboxPostControl() ?? workerMailboxPostCapability; - -/** - * The tool catalog: the scanned `workforce/blocks/` map, which holds - * `escalate`. The goal check's `no-filing` control swaps `escalate` for a - * stand-in that files nothing, in test mode only. - */ -const catalog = escalateControl(blocks) ?? blocks; +import { blocks, kinds, packageBlocks, seatBlocks } from "./workforce.gen"; /** This directory — the workforce root, read at run time the way Markdown always is. */ export const workforceRoot = dirname(fileURLToPath(import.meta.url)); -/** The installation, the copies it registers, the mailboxes the tree declared, and what the loader could not read. */ +/** The installation, the copies it registers, and what the loader could not read. */ export interface KitchenSinkWorkforce { /** The worker model: the standard workers, and the checks every session of a worker flow meets. */ installation: WorkerInstallation; - /** One copy per worker flow, and the roster flow. Register these. */ + /** One copy per worker flow (`agent` and `coordinator` among them), and the roster flow. Register these. */ copies: FlowInstance[]; - /** - * One instance per DISTINCT mailbox kind the tree selected — never one per - * mailbox. Register these beside the copies; a mailbox is a named session on - * the instance its `MAILBOX.md` selected. - */ - mailboxFlows: FlowInstance[]; - /** - * The mailbox records themselves, handed back so the caller can open them - * once the runtime exists. Opening needs a session client, and there is no - * session client until the FlowState is built — so this module builds the - * instances and the caller opens the sessions. - */ - mailboxes: MailboxManifest[]; /** * Everything the loader could not read, by path — from **all three** of its * error channels, not just the worker one. A worker folder that failed is a @@ -94,9 +60,8 @@ export interface KitchenSinkWorkforce { * instructions its author wrote. This app's answer to all three is the same: * say so. * - * **A mailbox or a package that failed to load is NOT here.** Each throws - * instead: a mailbox short is a team with nowhere to talk, and a worker short - * a package runs without the instructions and tools its author gave it. + * **A package that failed to load is NOT here.** It throws instead: a worker + * short a package runs without the instructions and tools its author gave it. */ errors: string[]; } @@ -104,23 +69,16 @@ export interface KitchenSinkWorkforce { /** * Read the team's files and build the workforce they describe. * - * @returns The installation, its copies and mailbox flows, and any folder the loader reported. - * @throws If a mailbox or a package fails to load, or a worker flow is refused. + * @returns The installation, its copies, and any folder the loader reported. + * @throws If a package fails to load, or a worker flow or a worker is refused. */ export async function buildKitchenSinkWorkforce(): Promise { - const { workers, errors, skillErrors, teamErrors, packageErrors } = await readWorkforce(workforceRoot); - const read = await readMailboxesDirectory(workforceRoot); - const mailboxErrors = read.errors; - // The goal checks' `no-route` control takes the `routing:` lines off here, - // before anything is bound, in test mode only. - const mailboxes = withMailboxRouteControl(read.mailboxes); + const read = await readWorkforce(workforceRoot); + const { errors, skillErrors, teamErrors, packageErrors } = read; + // The goal checks' route and rounds controls change a coordinator's + // settings here, before anything is built, in test mode only. + const workers = coordinatorWorkersUnderControl(read.workers); - if (mailboxErrors.length > 0) { - throw new Error( - `mailboxes: ${mailboxErrors.length} mailbox(es) failed to load\n` + - mailboxErrors.map(({ path, error }) => ` ${path}: ${error.message}`).join("\n"), - ); - } if (packageErrors.length > 0) { throw new Error( `packages: ${packageErrors.length} package problem(s)\n` + @@ -139,37 +97,29 @@ export async function buildKitchenSinkWorkforce(): Promise seatBlocks, packageBlocks, }); - // The generated flows, plus the built-in `agent` carrying the post tool and - // the tool catalog. The goal check's `no-landing` control swaps it for one - // that answers with the same tools and lands no text answer, in test mode only. + // The built-in `agent`, carrying the tool catalog, takes the coordinator's + // posts. The goal check's `no-landing` control swaps it for one that takes a + // post and hands no answer back, in test mode only. + const agent = landingControl(blocks, installation) ?? defineAgentWorkerFlow({ installation, catalog: blocks }); flows = { ...kinds, - agent: - mailboxLandingControl(catalog, mailboxPost, installation) ?? - defineAgentWorkerFlow({ installation, uses: [mailboxPost], catalog }), + agent, + // The coordinator flow: `support.help` runs on it. Its delegates take + // posts on `agent`, and best fit's one evaluator call runs on ROUTE_MODEL. + // Its judgment turn, for a post best fit can't place, is the agent's own + // turn, built from the same catalog. + [COORDINATOR_KIND]: defineCoordinatorFlow({ + installation, + delegateFlows: delegateFlowsUnderControl([agent]), + routeModel: ROUTE_MODEL, + agent: { catalog: blocks }, + }), }; - // `mailboxBoards` is what lets the build say which declared board no copy - // drains. Without it the check has no roster to read and says nothing — - // which is the silence this app ships to make visible. - const copies = hireWorkforce(installation, { mailboxBoards: mailboxBoardIds(mailboxes) }); - - // The generated map, plus the built-in under the key the binder seeds. The - // fan-out wakes each standard worker the mailbox names, on its flow's copy. - const mailboxFlows = mailboxInstances(mailboxes, { - kinds: { - ...mailboxKinds, - mailbox: defineMailboxFlow({ - notify: notifyFor(copies, installation), - route: routeByPurpose(copies, { model: ROUTE_MODEL, installation }), - }), - }, - }); + const copies = hireWorkforce(installation); return { installation, copies, - mailboxFlows, - mailboxes, errors: [ ...errors.map((e) => e.path), ...skillErrors.flatMap((seat) => seat.errors.map((e) => e.path)), diff --git a/apps/kitchen-sink/workforce/mailbox-notify.ts b/apps/kitchen-sink/workforce/mailbox-notify.ts deleted file mode 100644 index ddaca57400..0000000000 --- a/apps/kitchen-sink/workforce/mailbox-notify.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * The fan-out slot the built-in mailbox kind is built with. - * - * Here rather than under `flows/mailboxes/`, and that is the point: this is a - * block handed to the framework's own factory, not a mailbox kind of this - * app's own. `fsdev gen` walks `flows/mailboxes/` and would have put a file - * there on the generated `mailboxKinds` map, where the binder would have read - * it as a kind a `MAILBOX.md` can name in its `flow:` line. The top level of - * `workforce/` is not walked, so a block that belongs to the built-in lives - * here and stays off that map. - * - * The wake is Workforce's `wakeMemberSeats`, over the copies registered at - * boot, with nothing of this app's added: every member of `support.help` is a - * standard worker on `agent`, which hears posts, so there is no member to - * stand in for. Each member wakes in a conversation of its own on the one - * `agent` copy. On a routed mailbox the route has already picked who hears a - * post, and the wake runs that worker alone. A worker's own post wakes nobody. - * The goal checks' two wake controls wrap it here, from - * `lib/mailbox-wake-control.ts`, and never reach the package. - */ -import type { BlockDefinition } from "@flow-state-dev/core"; -import type { FlowInstance } from "@flow-state-dev/core/types"; -import { wakeMemberSeats, type WorkerInstallation } from "@flow-state-dev/workforce"; - -import { withMailboxWakeControl } from "../lib/mailbox-wake-control"; - -/** - * The notify block: the wake for each member that can hear a post. - * - * @param copies The copies registered at boot. - * @param installation The installation whose standard workers the members are. - * @returns The block `defineMailboxFlow({ notify })` runs once per member per post. - */ -export function notifyFor(copies: readonly FlowInstance[], installation: WorkerInstallation): BlockDefinition { - return withMailboxWakeControl(wakeMemberSeats(copies, { installation })); -} diff --git a/apps/kitchen-sink/workforce/teams/support/mailboxes/help/MAILBOX.md b/apps/kitchen-sink/workforce/teams/support/mailboxes/help/MAILBOX.md deleted file mode 100644 index 51590cfd48..0000000000 --- a/apps/kitchen-sink/workforce/teams/support/mailboxes/help/MAILBOX.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -description: Ask the support team anything. -members: [support.devices, support.accounts, support.fsd, support.general] -boards: [escalations] -boardActions: true -routing: - fallback: support.general ---- - -Ask here. The specialist whose subject fits your post answers it under its own name. A case -that needs a person goes on `escalations`, and nobody here works that board yet. diff --git a/apps/kitchen-sink/workforce/teams/support/workers/accounts/WORKER.md b/apps/kitchen-sink/workforce/teams/support/workers/accounts/WORKER.md index be635f6478..62d7bf037d 100644 --- a/apps/kitchen-sink/workforce/teams/support/workers/accounts/WORKER.md +++ b/apps/kitchen-sink/workforce/teams/support/workers/accounts/WORKER.md @@ -1,7 +1,6 @@ --- description: Sign-in, passwords, billing, refunds and subscriptions. -tools: [post-to-mailbox, escalate] --- You are the support team's accounts specialist. Answer in a sentence or two, and say plainly -when you don't know. When a case needs a person, file it with `escalate` and say you did. +when you don't know. diff --git a/apps/kitchen-sink/workforce/teams/support/workers/devices/WORKER.md b/apps/kitchen-sink/workforce/teams/support/workers/devices/WORKER.md index c6748cddcf..3f7f3b5353 100644 --- a/apps/kitchen-sink/workforce/teams/support/workers/devices/WORKER.md +++ b/apps/kitchen-sink/workforce/teams/support/workers/devices/WORKER.md @@ -1,7 +1,6 @@ --- description: Printers, laptops, phones, wifi and anything else with a power button. -tools: [post-to-mailbox, escalate] --- You are the support team's devices specialist. Answer in a sentence or two, and say plainly -when you don't know. When a case needs a person, file it with `escalate` and say you did. +when you don't know. diff --git a/apps/kitchen-sink/workforce/teams/support/workers/fsd/WORKER.md b/apps/kitchen-sink/workforce/teams/support/workers/fsd/WORKER.md index a18bc7f2b0..d39949526e 100644 --- a/apps/kitchen-sink/workforce/teams/support/workers/fsd/WORKER.md +++ b/apps/kitchen-sink/workforce/teams/support/workers/fsd/WORKER.md @@ -1,7 +1,6 @@ --- description: Questions about building apps with the flow-state-dev framework. -tools: [post-to-mailbox, escalate] --- You are the support team's flow-state-dev specialist. Answer in a sentence or two, and say -plainly when you don't know. When a case needs a person, file it with `escalate` and say you did. +plainly when you don't know. diff --git a/apps/kitchen-sink/workforce/teams/support/workers/general/WORKER.md b/apps/kitchen-sink/workforce/teams/support/workers/general/WORKER.md index c0f0764acf..0d46fa190a 100644 --- a/apps/kitchen-sink/workforce/teams/support/workers/general/WORKER.md +++ b/apps/kitchen-sink/workforce/teams/support/workers/general/WORKER.md @@ -1,8 +1,6 @@ --- description: Anything that fits none of the other specialists. -tools: [post-to-mailbox, escalate] --- You are the support team's general specialist: you take what the others don't. Answer in a -sentence or two, and say plainly when you don't know. When a case needs a person, file it with -`escalate` and say you did. +sentence or two, and say plainly when you don't know. diff --git a/apps/kitchen-sink/workforce/teams/support/workers/help/WORKER.md b/apps/kitchen-sink/workforce/teams/support/workers/help/WORKER.md new file mode 100644 index 0000000000..b89ffa209c --- /dev/null +++ b/apps/kitchen-sink/workforce/teams/support/workers/help/WORKER.md @@ -0,0 +1,9 @@ +--- +description: Ask the support team anything. +flow: coordinator +delegates: [support.devices, support.accounts, support.fsd, support.general] +routing: best-fit +fallback: support.general +--- + +Ask here. The specialist whose subject fits your post answers it under its own name. diff --git a/apps/kitchen-sink/workforce/workforce.gen.ts b/apps/kitchen-sink/workforce/workforce.gen.ts index 697749bcde..72faa0f53a 100644 --- a/apps/kitchen-sink/workforce/workforce.gen.ts +++ b/apps/kitchen-sink/workforce/workforce.gen.ts @@ -8,16 +8,11 @@ // this file and the tree disagree. Committed on purpose — a fresh clone has to // typecheck before anything has been generated. -import type { BlockDefinition } from "@flow-state-dev/core"; -import block_escalate from "./blocks/escalate"; - export const kinds = {}; export const mailboxKinds = {}; -export const blocks = { - "escalate": block_escalate, -} satisfies Record; +export const blocks = {}; export const resourceModules = {}; diff --git a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/fixtures/workforce/teams/desk/mailboxes/front/MAILBOX.md b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/fixtures/workforce/teams/desk/mailboxes/front/MAILBOX.md deleted file mode 100644 index c520522311..0000000000 --- a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/fixtures/workforce/teams/desk/mailboxes/front/MAILBOX.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -description: Where the desk talks. -members: [desk.amy, desk.oz, desk.ned] ---- - -One line per question. diff --git a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/fixtures/workforce/teams/desk/workers/front/WORKER.md b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/fixtures/workforce/teams/desk/workers/front/WORKER.md new file mode 100644 index 0000000000..8cab5a3ca7 --- /dev/null +++ b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/fixtures/workforce/teams/desk/workers/front/WORKER.md @@ -0,0 +1,8 @@ +--- +description: Where the desk talks. +flow: coordinator +delegates: [desk.amy, desk.oz, desk.ned] +routing: everyone +--- + +One line per question. diff --git a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md index ddf2912ce8..446f5baaed 100644 --- a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md +++ b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md @@ -1,31 +1,34 @@ # workforce-mailboxes › a fresh host wakes its member agents -**Issue:** FIX-1602 (VG of the spec's PLAN; feeds the epic FIX-1592's closure, FIX-1601) +**Issue:** FIX-1602 (VG of the spec's PLAN; feeds the epic FIX-1592's closure, FIX-1601). Converted by FIX-1792 (BR-9): the mailbox is a coordinator's worker file now. -**Outcome:** An app with none of kitchen-sink's code wakes each agent seat in a mailbox once per post, and never on a seat's own post, by putting one Workforce call in the mailbox's notify slot. Kitchen-sink runs on that same call. +**Outcome:** An app with none of kitchen-sink's code hands each post in a person's conversation with a coordinator to every agent delegate once, and a delegate's answer wakes nobody, by declaring the coordinator in a `WORKER.md` (`flow: coordinator`, `routing: everyone`) and registering Workforce's `coordinator` flow. Kitchen-sink runs on that same flow. -**Input:** `fixtures/workforce/`: one team, two agent seats (no `flow:`), one seat of the app's own `note` kind (only a public action, no `onMailboxPost`), and one mailbox naming all three. `host.mts` is the app: the tree, the published packages, and `wakeMemberSeats(seats)`. Held-out: who is an agent member is read off the tree (a worker with no `flow:`), never hardcoded, and each run's posts carry fresh tokens. Renaming the team, the mailbox and every worker, and reordering `members:`, passed unchanged. +**Converted (FIX-1792).** The folder name is kept so its verdict history stays in one place. Until FIX-1792 this read a `MAILBOX.md` whose post woke each agent member through `wakeMemberSeats` in the mailbox's notify slot. The mailbox is a coordinator now: `members:` became `delegates:`, and a file with no `routing:` line got `routing: everyone`, which is what its mailbox did. The **seat-post** leg (a member's own post wakes nobody) has no door to post through on a coordinator; its outcome, that what a worker says wakes nobody, is now **quiet** (a delegate's answer wakes nobody), and its control flips the setting that holds it. -**Signal:** the host served in-process on in-memory stores and a scripted model. Two person posts through the public action, then one post on the mailbox's internal seat entry (the door a seat's own post uses), carrying that member as `author`. Everything graded is read back through the host's own router, the routes a page reads: each seat's conversations, dispatch runs included. +**Input:** `fixtures/workforce/`: one team, two agent workers (no `flow:`), one worker of the app's own `note` flow (only a public action, no delegated-post entry), and one coordinator naming all three as delegates. `host.mts` is the app: the tree, the published packages, and the `coordinator` flow registered with the built-in `agent` as the flow its delegates take posts on. Held-out: who is an agent delegate is read off the tree (a worker with no `flow:`), never hardcoded, and each run's posts carry fresh tokens. -- **import**: `wakeMemberSeats` resolves from `@flow-state-dev/workforce`. Checked first; the rest needs it. -- **woken**: each agent member holds exactly one conversation of the mailbox, and in it each person's post is heard in exactly one `user` turn with an `assistant` reply right under it. -- **seat-post**: no seat's conversation holds the agent's own post. -- **other**: the `note` member holds no conversation. -- **source**: `host.mts` imports only `@flow-state-dev/*`, calls `wakeMemberSeats`, and builds no `dispatcher`, `keyedRouter` or `router`. Kitchen-sink's `workforce/mailbox-notify.ts` calls `wakeMemberSeats` and builds none of the three, and its `SEAT_ASKS` has no `wake` column. +**Signal:** the host served in-process on in-memory stores and a scripted model. The person opens a conversation with the coordinator the way a page does (a session on `coordinator` naming it) and posts twice through its public `run` action. Everything graded is read back through the host's own router, the routes a page reads: the conversation's items, and each worker's conversations, dispatch runs included. -**Anti-game:** no assertion on the router's output, a dispatch handle, a router decision, or a unit test. Only what each seat's own conversation kept counts, read through the routes, and only this run's tokens. The wait before grading polls until the agents answer and then gives a wrongly woken seat 1.5s to run; none of it is graded. Kitchen-sink's runtime half is not re-proved here: FIX-1590's `a-post-runs-each-member-agent-once` and FIX-1594's `agent-replies-in-the-mailbox` are re-run on the thinned app. +- **import**: `defineCoordinatorFlow` resolves from `@flow-state-dev/workforce`. Checked first; the rest needs it. +- **woken**: each agent delegate holds exactly one conversation under the person's, and in it each post is heard in exactly one `user` turn with an `assistant` reply right under it. +- **answers**: the person's conversation holds exactly one answer line per post from each agent delegate, under its name, and none by anyone else. +- **quiet**: no worker's conversation holds a delegate's answer as a turn it heard. +- **other**: the `note` delegate holds no conversation. +- **source**: `host.mts` imports only `@flow-state-dev/*`, calls `defineCoordinatorFlow`, and builds no `dispatcher`, `keyedRouter` or `router`; the tree's coordinator routes to `everyone`. Kitchen-sink's `workforce/hire.ts` calls `defineCoordinatorFlow` and builds none of the three, and its `SEAT_ASKS` has no `wake` column. -**Model:** n/a. The built-in agent kind answers from a scripted model (`@flow-state-dev/testing`), keyless (epic FIX-1592 D3). The goal is who ran, not what they said. +**Anti-game:** no assertion on the router's output, a dispatch handle, a routing record, or a unit test. Only what each worker's own conversation kept and what the person's conversation holds count, read through the routes, and only this run's tokens. The wait before grading polls until every request has settled and then gives a wrongly woken worker 1.5s to run; none of it is graded. Kitchen-sink's runtime half is not re-proved here: `kitchen-sink-talk`'s goals are re-run on the converted app. + +**Model:** n/a. The built-in agent flow answers from a scripted model (`@flow-state-dev/testing`), keyless (epic FIX-1592 D3). The goal is who ran, not what they said. **Run:** `pnpm --dir goals exec tsx workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts` -**Controls:** on the same command. Each is applied by the check around the host's wake, never inside the package. +**Controls:** on the same command. Each is applied by the check around the host, never inside the package. -- `GOAL_CONTROL=no-wake`: the mailbox gets no notify block. Must FAIL at **woken**, and at nothing else. -- `GOAL_CONTROL=no-author-filter`: the host strips `seatAuthored` from each delivery before the helper sees it. The agent's own post then wakes both agents. Must FAIL at **seat-post**, and at nothing else. +- `GOAL_CONTROL=no-wake`: the coordinator flow is registered with no flow a delegate takes a post on. Must FAIL at **woken** and **answers**, and at nothing else. +- `GOAL_CONTROL=answers-go-on`: the coordinator's file is read with `rounds: 1`, so each round's answers go back out to the other delegates. Must FAIL at **answers** (each delegate answers the other's answer too) and **quiet**, and at nothing else. -The **import** and **source** legs have no control: a checkout without the export fails **import** (as `main` did before this issue), and kitchen-sink's pre-move `mailbox-notify.ts` and `workforce-shell.ts` fail **source**. Both were produced by hand and are logged below. +The **import** and **source** legs have no control: a checkout without the export fails **import**, and kitchen-sink's pre-conversion `workforce/hire.ts` fails **source**. Both were produced and are logged below. ## Verdict log | Date | Commit | Model | Verdict | Notes | diff --git a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/host.mts b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/host.mts index 162cc30700..1d76112095 100644 --- a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/host.mts +++ b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/host.mts @@ -1,39 +1,36 @@ /** - * A fresh app with a mailbox of agents, as a new app would write it. + * A fresh app with a coordinator over agents, as a new app would write it. * * Everything here comes from the published packages and the team's files: no - * kitchen-sink code, and no dispatcher or router of the app's own. The wake is - * one call, `wakeMemberSeats(copies, { installation })`, in the built-in - * mailbox kind's notify slot. The goal check reads this file's source to hold - * it to that. + * kitchen-sink code, and no dispatcher or router of the app's own. The + * coordinator is a worker file (`flow: coordinator`, `routing: everyone`), + * and the app registers Workforce's `coordinator` flow once, with the + * built-in `agent` as the flow its delegates take posts on. The goal check + * reads this file's source to hold it to that. * * The app has one worker flow of its own, `note`, whose workers take notes - * when asked and declare no `onMailboxPost`, so a post runs nothing on them. - * Every other worker runs on the built-in `agent`, answered by a scripted - * model so the check needs no key. Each flow is registered once; a woken - * worker's conversation is a session on its flow, naming it. + * when asked and declare no delegated-post entry, so a post runs nothing on + * them. Every other worker runs on the built-in `agent`, answered by a + * scripted model so the check needs no key. Each flow is registered once; a + * delegate's conversation is a session on its flow, naming it. * - * `adaptNotify` is the goal check's seam for its controls, and nothing else: - * given the wake this app builds, it returns the block the mailbox runs, or - * `undefined` for no notify block at all. An app passes nothing. + * `seams` is the goal check's, for its controls, and nothing else. An app + * passes nothing. */ -import { createSessionClient } from "@flow-state-dev/client"; -import { defineFlow, handler, type BlockDefinition } from "@flow-state-dev/core"; +import { defineFlow, handler } from "@flow-state-dev/core"; import type { FlowInstance } from "@flow-state-dev/core/types"; import { createFlowState, inMemoryStores, type FlowState } from "@flow-state-dev/engine"; import { createMockModelResolver, mockGenerator } from "@flow-state-dev/testing"; import { - mailboxInstances, createWorkerInstallation, - defineMailboxFlow, + defineAgentWorkerFlow, + defineCoordinatorFlow, hireWorkforce, - openMailboxes, - wakeMemberSeats, workerConfigSchema, - type MailboxManifest, - type WorkerInstallation + type WorkerInstallation, + type WorkerManifest } from "@flow-state-dev/workforce"; -import { readMailboxesDirectory, readWorkforce } from "@flow-state-dev/workforce/loader"; +import { readWorkforce } from "@flow-state-dev/workforce/loader"; /** * `{ message: string }`, built off the contract's own `instructions` string so @@ -57,13 +54,20 @@ const workerDoor = { }, }; -/** The user the mailboxes are opened under, and who posts to them. */ -export const MAILBOX_OWNER = "u_fresh_host"; +/** The user who talks to the coordinator. */ +export const OWNER = "u_fresh_host"; -/** What an agent seat answers with, whatever it heard. */ +/** What an agent delegate answers with, whatever it heard. */ export const REPLY_MARKER = "[reply:fresh-host]"; -/** This app's own worker flow: takes a note when asked directly, and hears no posts. */ +/** + * The model best fit's one evaluator call would run on. This app's + * coordinator routes to everyone and never makes that call, but the flow + * names no default, so the app names one. + */ +const ROUTE_MODEL = "vercel/typesafe-ai/jev"; + +/** This app's own worker flow: takes a note when asked directly, and takes no posts. */ const defineNote = (installation: WorkerInstallation) => defineFlow({ kind: "note", @@ -74,50 +78,49 @@ const defineNote = (installation: WorkerInstallation) => actions: { ...workerDoor, take: { block: handler({ name: "note-take", execute: () => ({}) }) } } } as never); -/** The app, booted: its router, what it hired, and a way to shut it down. */ +/** The goal check's seams. An app passes none. */ +export interface HostSeams { + /** The worker files as the host reads them, before anything is built. */ + adaptWorkers?: (workers: WorkerManifest[]) => WorkerManifest[]; + /** The flows a delegate takes a post on, given the ones the app names. */ + adaptDelegateFlows?: (flows: { kind: string }[]) => { kind: string }[]; +} + +/** The app, booted: its router, and what it registered. */ export interface FreshHost { state: FlowState; router: Awaited>; - seats: FlowInstance[]; - mailboxes: MailboxManifest[]; - /** The built mailbox kind, so a check can post on the internal seat entry. */ - mailbox: FlowInstance; + copies: FlowInstance[]; } /** - * Read the team's files, hire its seats, build its mailboxes with the wake in - * the notify slot, and open them. + * Read the team's files, register one copy of each worker flow, the + * coordinator's among them, and serve them. * * @param tree The workforce root to read. - * @param adaptNotify The goal check's control seam. An app passes nothing. + * @param seams The goal check's seams. An app passes nothing. */ -export async function startFreshHost( - tree: string, - adaptNotify?: (wake: BlockDefinition) => BlockDefinition | undefined -): Promise { - const { workers, errors } = await readWorkforce(tree); - const { mailboxes, errors: mailboxErrors } = await readMailboxesDirectory(tree); - if (errors.length > 0 || mailboxErrors.length > 0) { - throw new Error(`the tree did not load: ${[...errors, ...mailboxErrors].map((e) => e.path).join(", ")}`); +export async function startFreshHost(tree: string, seams: HostSeams = {}): Promise { + const read = await readWorkforce(tree); + if (read.errors.length > 0) { + throw new Error(`the tree did not load: ${read.errors.map((e) => `${e.path}: ${String(e.error)}`).join(", ")}`); } + const workers = seams.adaptWorkers?.(read.workers) ?? read.workers; - // The flows first: the wake reaches the workers on these copies, never a - // mailbox's stored members. + // The flows are handed to the installation lazily, so each can be built on it. let flows: Record = {}; const installation = createWorkerInstallation({ standardWorkers: workers, workerFlows: () => flows as never }); - flows = { note: defineNote(installation) }; - const seats = hireWorkforce(installation); - const wake = wakeMemberSeats(seats, { installation }); - const notify = adaptNotify === undefined ? wake : adaptNotify(wake); - const mailboxFlows = mailboxInstances(mailboxes, { - kinds: { mailbox: notify === undefined ? defineMailboxFlow() : defineMailboxFlow({ notify }) } - }); + const agent = defineAgentWorkerFlow({ installation }); + const delegateFlows = seams.adaptDelegateFlows?.([agent]) ?? [agent]; + flows = { + agent, + coordinator: defineCoordinatorFlow({ installation, delegateFlows: delegateFlows as never, routeModel: ROUTE_MODEL }), + note: defineNote(installation) + }; + const copies = hireWorkforce(installation); const state = createFlowState({ - flows: { - ...Object.fromEntries(mailboxFlows.map((flow) => [flow.kind, flow])), - ...Object.fromEntries(seats.map((seat) => [seat.id, seat])) - }, + flows: Object.fromEntries(copies.map((copy) => [copy.id, copy])), stores: { default: { primary: inMemoryStores() } }, modelResolver: createMockModelResolver({ generators: { @@ -130,21 +133,5 @@ export async function startFreshHost( }) } as never); const router = await state.getRouter(); - - // The session client over the app's own router: the app is the server. - const sessions = createSessionClient({ - fetcher: async (input, init) => { - const url = new URL(String(input), "http://fresh-host.local"); - const path = url.pathname - .replace(/^\/api\/flows\/?/, "") - .split("/") - .filter((segment) => segment.length > 0) - .map(decodeURIComponent); - const method = (init?.method ?? "GET").toUpperCase() as "GET" | "POST" | "PATCH" | "DELETE"; - return await router[method](new Request(url, init), { params: { path } }); - } - }); - await openMailboxes(mailboxes, { client: sessions, userId: MAILBOX_OWNER }); - - return { state, router, seats, mailboxes, mailbox: mailboxFlows[0]! }; + return { state, router, copies }; } diff --git a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts index 874c6be610..55030af404 100644 --- a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts +++ b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts @@ -1,39 +1,45 @@ /** - * Goal check: a host with none of kitchen-sink's code wakes each member agent - * seat once per post, and never on a seat's own post, by calling one Workforce - * helper, and kitchen-sink runs on that same helper. + * Goal check: a host with none of kitchen-sink's code hands each post in a + * coordinator conversation to every agent delegate once, and a delegate's + * answer wakes nobody, by declaring the coordinator in a worker file + * (`flow: coordinator`, `routing: everyone`) and registering Workforce's + * coordinator flow; kitchen-sink runs on that same flow. * * Real path, scripted model, out of CI. See goal.md for the contract. * * The host is `host.mts`, an app built from the published packages and the * fixture tree alone. It is served in-process, and everything graded is read - * back through its own router, the routes a page reads: each seat's - * conversations, dispatch runs included. Legs: + * back through its own router, the routes a page reads: the coordinator + * conversation's items and each delegate's conversations, dispatch runs + * included. Legs: * - * import `wakeMemberSeats` resolves from `@flow-state-dev/workforce`. - * woken each agent member holds one conversation of the mailbox, both - * person posts heard once in it, each with a reply under it. - * seat-post a post an agent member wrote is heard by no seat. - * other the member whose kind declares no `onMailboxPost` holds nothing. - * source the host imports only `@flow-state-dev/*` and builds no - * dispatcher or router; kitchen-sink's notify module calls the - * helper and builds neither, and its kind map has no wake column. + * import `defineCoordinatorFlow` resolves from `@flow-state-dev/workforce`. + * woken each agent delegate holds one conversation under the person's, + * both posts heard once in it, each with a reply under it. + * answers the person's conversation holds one answer line per post from + * each agent delegate, under its name. + * quiet no delegate hears another's answer. + * other the delegate whose flow takes no posts holds nothing. + * source the host imports only `@flow-state-dev/*`, registers the + * coordinator flow and builds no dispatcher or router, and the + * tree's coordinator routes to everyone; kitchen-sink registers + * the same flow and builds neither, and its kind map has no wake + * column. * - * Who is an agent member is read off the tree (a worker with no `flow:`), so - * renamed folders grade the same way. + * Who is an agent delegate is read off the tree (a worker with no `flow:`), + * so renamed folders grade the same way. * - * Run: pnpm tsx goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts - * Controls: GOAL_CONTROL=no-wake (no notify block: must FAIL at woken, and nothing else) - * GOAL_CONTROL=no-author-filter (seat mark stripped before the helper: must FAIL at seat-post, and nothing else) + * Run: pnpm --dir goals exec tsx workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts + * Controls: GOAL_CONTROL=no-wake (no flow takes a delegated post: must FAIL at woken and answers, and nothing else) + * GOAL_CONTROL=answers-go-on (the coordinator read with `rounds: 1`: must FAIL at answers and quiet, and nothing else) */ import { randomUUID } from "node:crypto"; import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; -import { DEFAULT_ORG_ID, type BlockDefinition } from "@flow-state-dev/core"; -import { runAction } from "@flow-state-dev/engine"; -import type { MailboxNotifyInput } from "@flow-state-dev/workforce"; -import { readMailboxesDirectory, readWorkforce } from "@flow-state-dev/workforce/loader"; +import type { WorkerManifest } from "@flow-state-dev/workforce"; +import { readWorkforce } from "@flow-state-dev/workforce/loader"; import { KITCHEN_SINK, runGoal } from "../../lib/index.mts"; +import type { FreshHost, HostSeams } from "./host.mts"; const TREE = fileURLToPath(new URL("./fixtures/workforce", import.meta.url)); const HOST_SOURCE = fileURLToPath(new URL("./host.mts", import.meta.url)); @@ -41,72 +47,104 @@ const CONTROL = process.env.GOAL_CONTROL ?? ""; /** The legs each control must redden, and only those. */ const EXPECTED: Record = { - // No notify block: a post reaches no seat. - "no-wake": ["woken"], - // The host strips `seatAuthored` before the helper: an agent's own post wakes the agents. - "no-author-filter": ["seat-post"], + // No flow takes a delegated post: the post reaches no delegate, and nothing answers. + "no-wake": ["woken", "answers"], + // Each round's answers go back out: each delegate hears the other's answer and answers it. + "answers-go-on": ["answers", "quiet"], }; if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); } -/** The control's change to the wake the host builds, or none. */ -function adaptNotify(wake: BlockDefinition): BlockDefinition | undefined { - if (CONTROL === "no-wake") return undefined; - if (CONTROL === "no-author-filter") { - return wake.connectInput(({ seatAuthored: _seatAuthored, ...post }: MailboxNotifyInput) => post); +/** The control's seams on the host, or none. */ +function controlSeams(): HostSeams { + switch (CONTROL) { + case "no-wake": + return { adaptDelegateFlows: () => [] }; + case "answers-go-on": + return { + adaptWorkers: (workers: WorkerManifest[]) => + workers.map((w) => (w.declared.flow === "coordinator" ? { ...w, declared: { ...w.declared, rounds: 1 } } : w)), + }; + default: + return {}; } - return wake; } type Message = { role: string; text: string }; type Conversation = { id: string; parentSessionId?: string; messages: Message[] }; +/** Until every request in the host has settled, three reads running. Not graded. */ +async function quiet(app: FreshHost, ms: number): Promise { + const runtime = await app.state.getRuntime(); + const busy = async () => { + for (const session of await runtime.stores.session.list({ parentage: "all" })) { + const requests = await runtime.stores.request.list({ sessionId: session.id }); + if (requests.some((r) => r.status === "in_progress")) return true; + } + return false; + }; + const until = Date.now() + ms; + for (let calm = 0; calm < 3 && Date.now() < until; ) { + calm = (await busy()) ? 0 : calm + 1; + await new Promise((r) => setTimeout(r, 50)); + } +} + await runGoal(async () => { const failures: string[] = []; const evidence: string[] = []; const fail = (leg: string, line: string) => failures.push(`[${leg}] ${line}`); - // ---- import: the helper is there to call -------------------------------- + // ---- import: the flow is there to register -------------------------------- const workforce = (await import("@flow-state-dev/workforce")) as Record; - if (typeof workforce.wakeMemberSeats !== "function") { - fail("import", "@flow-state-dev/workforce exports no wakeMemberSeats"); + if (typeof workforce.defineCoordinatorFlow !== "function") { + fail("import", "@flow-state-dev/workforce exports no defineCoordinatorFlow"); return { failures, evidence: "" }; } - evidence.push("wakeMemberSeats resolves from @flow-state-dev/workforce"); + evidence.push("defineCoordinatorFlow resolves from @flow-state-dev/workforce"); + + // ---- the tree: the coordinator, and who should hear a post ------------------ + const { workers } = await readWorkforce(TREE); + const coordinators = workers.filter((w) => w.declared.flow === "coordinator"); + const coordinator = coordinators[0]; + if (coordinators.length !== 1 || coordinator === undefined) { + throw new Error(`the fixture needs one coordinator; read ${coordinators.map((w) => w.id).join(", ") || "none"}`); + } + const delegates = (coordinator.declared.delegates ?? []) as string[]; + const agents = delegates.filter((id) => !Object.hasOwn(workers.find((w) => w.id === id)?.declared ?? {}, "flow")); + const others = delegates.filter((id) => !agents.includes(id)); + if (agents.length < 2 || others.length < 1) { + throw new Error(`the fixture needs two agent delegates and one other; read ${agents.join(", ")} / ${others.join(", ")}`); + } - // ---- source: what the host and kitchen-sink write ------------------------ + // ---- source: what the host and kitchen-sink write --------------------------- const builds = /\b(dispatcher|keyedRouter|router)\s*\(/; const host = readFileSync(HOST_SOURCE, "utf8"); const imports = [...host.matchAll(/^import[^;]*?from\s+"([^"]+)"/gms)].map((m) => m[1]!); const foreign = imports.filter((spec) => !spec.startsWith("@flow-state-dev/")); if (foreign.length > 0) fail("source", `host.mts imports from outside @flow-state-dev/*: ${foreign.join(", ")}`); if (builds.test(host)) fail("source", `host.mts builds its own ${builds.exec(host)![1]}`); - if (!/wakeMemberSeats\s*\(/.test(host)) fail("source", "host.mts never calls wakeMemberSeats"); - const notifyModule = readFileSync(`${KITCHEN_SINK}/workforce/mailbox-notify.ts`, "utf8"); - if (builds.test(notifyModule)) fail("source", `kitchen-sink's mailbox-notify.ts builds its own ${builds.exec(notifyModule)![1]}`); - if (!/wakeMemberSeats\s*\(/.test(notifyModule)) fail("source", "kitchen-sink's mailbox-notify.ts never calls wakeMemberSeats"); + if (!/defineCoordinatorFlow\s*\(/.test(host)) fail("source", "host.mts never registers defineCoordinatorFlow"); + if (coordinator.declared.routing !== "everyone") { + fail("source", `${coordinator.id} routes by ${String(coordinator.declared.routing ?? "its default")} (want everyone)`); + } + const hire = readFileSync(`${KITCHEN_SINK}/workforce/hire.ts`, "utf8"); + if (builds.test(hire)) fail("source", `kitchen-sink's workforce/hire.ts builds its own ${builds.exec(hire)![1]}`); + if (!/defineCoordinatorFlow\s*\(/.test(hire)) fail("source", "kitchen-sink's workforce/hire.ts never registers defineCoordinatorFlow"); const shell = readFileSync(`${KITCHEN_SINK}/lib/workforce-shell.ts`, "utf8"); const asks = /export const SEAT_ASKS = \{([\s\S]*?)\} as const/.exec(shell)?.[1]; if (asks === undefined) fail("source", "kitchen-sink's lib/workforce-shell.ts has no SEAT_ASKS to read"); else if (/\bwake\s*:/.test(asks)) fail("source", "kitchen-sink's SEAT_ASKS still carries a wake column"); if (!failures.some((f) => f.startsWith("[source]"))) { - evidence.push(`host.mts imports only ${[...new Set(imports)].join(", ")} and builds no dispatcher or router; kitchen-sink's notify module calls wakeMemberSeats and builds neither`); - } - - // ---- the tree: who should hear a post ------------------------------------ - const { workers } = await readWorkforce(TREE); - const { mailboxes } = await readMailboxesDirectory(TREE); - const mailbox = mailboxes[0]!; - const members = mailbox.declared.members as string[]; - const agents = members.filter((id) => !Object.hasOwn(workers.find((w) => w.id === id)?.declared ?? {}, "flow")); - const others = members.filter((id) => !agents.includes(id)); - if (agents.length < 2 || others.length < 1) { - throw new Error(`the fixture needs two agent members and one other; read ${agents.join(", ")} / ${others.join(", ")}`); + evidence.push( + `host.mts imports only ${[...new Set(imports)].join(", ")}, registers defineCoordinatorFlow and builds no dispatcher or router; ` + + `${coordinator.id} routes to everyone; kitchen-sink registers defineCoordinatorFlow and builds neither`, + ); } - const { startFreshHost, MAILBOX_OWNER, REPLY_MARKER } = await import("./host.mts"); - const app = await startFreshHost(TREE, adaptNotify); + const { startFreshHost, OWNER, REPLY_MARKER } = await import("./host.mts"); + const app = await startFreshHost(TREE, controlSeams()); const call = async (method: "GET" | "POST", segments: string[], body?: unknown, query = "") => { const res = await app.router[method]( new Request(`http://fresh-host.local/api/flows/${segments.map(encodeURIComponent).join("/")}${query}`, { @@ -118,13 +156,16 @@ await runGoal(async () => { ); return { status: res.status, text: await res.text() }; }; - const post = async (body: string, author?: string) => { - const input = author === undefined ? { body } : { body, author }; - const res = await call("POST", [String(mailbox.declared.flow ?? "mailbox"), "actions", "post"], { userId: MAILBOX_OWNER, sessionId: mailbox.id, input }); - if (res.status !== 200) throw new Error(`post "${body}" refused: ${res.status} ${res.text.slice(0, 300)}`); + /** The person's conversation with the coordinator, opened the way a page opens one. */ + const opened = await call("POST", ["coordinator", "sessions"], { userId: OWNER, state: { workerId: coordinator.id } }); + if (opened.status !== 201) throw new Error(`the conversation was not opened: ${opened.status} ${opened.text.slice(0, 300)}`); + const conversation = (JSON.parse(opened.text) as { session: { id: string } }).session.id; + const post = async (message: string) => { + const res = await call("POST", ["coordinator", "actions", "run"], { userId: OWNER, sessionId: conversation, input: { message } }); + if (res.status !== 200) throw new Error(`post "${message}" refused: ${res.status} ${res.text.slice(0, 300)}`); }; const conversationsOf = async (seat: string): Promise => { - const listed = await call("GET", ["sessions"], undefined, `?flowId=agent&state.workerId=${encodeURIComponent(seat)}&userId=${MAILBOX_OWNER}&include=dispatch-runs&limit=100`); + const listed = await call("GET", ["sessions"], undefined, `?flowId=agent&state.workerId=${encodeURIComponent(seat)}&userId=${OWNER}&include=dispatch-runs&limit=100`); const rows = (JSON.parse(listed.text) as { sessions?: Array<{ id: string; parentSessionId?: string | null }> }).sessions ?? []; const out: Conversation[] = []; for (const row of rows) { @@ -140,51 +181,31 @@ await runGoal(async () => { } return out; }; - /** Poll until `done`, up to `ms`. Not graded: whatever did not happen fails below. */ - const settle = async (done: () => Promise, ms: number) => { - const until = Date.now() + ms; - while (Date.now() < until && !(await done())) await new Promise((r) => setTimeout(r, 50)); - }; - const answered = async (token: string) => { - for (const seat of agents) { - const heard = (await conversationsOf(seat)).flatMap((c) => c.messages); - if (!heard.some((m) => m.role === "assistant" && m.text.includes(REPLY_MARKER))) return false; - if (!heard.some((m) => m.text.includes(token))) return false; - } - return true; + /** The answer lines the person's conversation holds: each message a delegate's name is on. */ + const answerLines = async () => { + const state = await call("GET", ["sessions", conversation, "state"], undefined, "?include_items=true&item_types=message&limit=1000"); + const items = (JSON.parse(state.text) as { items?: Array<{ agentName?: string; transient?: boolean; content?: Array<{ text?: string }> }> }).items ?? []; + return items + .filter((item) => item.transient !== true && typeof item.agentName === "string") + .map((item) => ({ author: item.agentName!, text: (item.content ?? []).map((c) => c.text ?? "").join("") })); }; const run = randomUUID().replace(/-/g, "").slice(0, 10); const tokens = [`wake-token-a${run}`, `wake-token-b${run}`]; - const seatToken = `seat-token-${run}`; try { for (const token of tokens) { await post(`${token} can someone look at the refund queue?`); - await settle(() => answered(token), 5_000); + await quiet(app, 8_000); } - // An agent member speaks through `seatPost`, the door the seat's own - // post uses. A public or dispatched `post` is not that door. - const runtime = await app.state.getRuntime(); - const seatPost = await runAction({ - source: "internal", - orgId: DEFAULT_ORG_ID, - flow: app.mailbox, - actionName: "seatPost", - input: { body: `${seatToken} I looked; it is empty.`, author: agents[0] }, - userId: MAILBOX_OWNER, - sessionId: mailbox.id, - stores: runtime.stores, - runtimeConfig: { ...runtime.runtimeConfig }, - }); - if (seatPost.error !== undefined) throw new Error(`seat post refused: ${String(seatPost.error)}`); - // Give a wrongly woken seat time to run, so its absence is not a race. + // Give a wrongly woken delegate time to run, so its absence is not a race. await new Promise((r) => setTimeout(r, 1_500)); + await quiet(app, 8_000); - // ---- woken: one conversation each, both posts heard once, a reply to each + // ---- woken: one conversation each, both posts heard once, a reply to each -- for (const seat of agents) { - const runs = (await conversationsOf(seat)).filter((c) => c.parentSessionId === mailbox.id); + const runs = (await conversationsOf(seat)).filter((c) => c.parentSessionId === conversation); if (runs.length !== 1) { - fail("woken", `${seat} holds ${runs.length} conversations of ${mailbox.id} (want 1)`); + fail("woken", `${seat} holds ${runs.length} conversations under ${conversation} (want 1)`); continue; } const messages = runs[0]!.messages; @@ -197,19 +218,40 @@ await runGoal(async () => { fail("woken", `${seat} heard ${token} in ${at.length} turns (want 1, with a reply under it): ${JSON.stringify(messages.map((m) => `${m.role}: ${m.text}`))}`); } } - if (ok) evidence.push(`${seat}: one conversation of ${mailbox.id}, each person's post heard once and answered ("${messages.find((m) => m.role === "user")!.text}")`); + if (ok) evidence.push(`${seat}: one conversation under ${coordinator.id}'s, each post heard once and answered ("${messages.find((m) => m.role === "user")!.text}")`); } - // ---- seat-post: the agent's own post is heard by nobody ------------------ - const heardSeatPost: string[] = []; - for (const seat of members) { - const turns = (await conversationsOf(seat)).flatMap((c) => c.messages).filter((m) => m.text.includes(seatToken)); - if (turns.length > 0) heardSeatPost.push(`${seat} (${JSON.stringify(turns[0]!.text)})`); + // ---- answers: one line per post from each agent delegate, under its name --- + { + const lines = await answerLines(); + let ok = true; + for (const seat of agents) { + const by = lines.filter((line) => line.author === seat && line.text.includes(REPLY_MARKER)); + if (by.length !== tokens.length) { + ok = false; + fail("answers", `${conversation} holds ${by.length} answer line(s) by ${seat} (want ${tokens.length}, one per post)`); + } + } + const strangers = lines.filter((line) => !agents.includes(line.author)); + if (strangers.length > 0) { + ok = false; + fail("answers", `${conversation} holds answer lines by ${[...new Set(strangers.map((l) => l.author))].join(", ")}, which are not agent delegates`); + } + if (ok) evidence.push(`${conversation}: ${tokens.length} answer lines from each of ${agents.join(", ")}`); + } + + // ---- quiet: no delegate hears another's answer ----------------------------- + { + const heardAnswers: string[] = []; + for (const seat of delegates) { + const turns = (await conversationsOf(seat)).flatMap((c) => c.messages).filter((m) => m.role === "user" && m.text.includes(REPLY_MARKER)); + if (turns.length > 0) heardAnswers.push(`${seat} (${JSON.stringify(turns[0]!.text)})`); + } + if (heardAnswers.length > 0) fail("quiet", `a delegate's answer was heard by ${heardAnswers.join(", ")}`); + else evidence.push("no delegate heard another's answer"); } - if (heardSeatPost.length > 0) fail("seat-post", `${agents[0]}'s own post was heard by ${heardSeatPost.join(", ")}`); - else evidence.push(`${agents[0]}'s own post was heard by no seat`); - // ---- other: the member whose kind cannot hear a post holds nothing ------- + // ---- other: the delegate whose flow takes no posts holds nothing ----------- for (const seat of others) { const held = await conversationsOf(seat); if (held.length > 0) fail("other", `${seat} holds ${held.length} conversations (want 0)`); diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/mailboxes/help/MAILBOX.md b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/mailboxes/help/MAILBOX.md deleted file mode 100644 index 4643317559..0000000000 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/mailboxes/help/MAILBOX.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -description: Ask support anything. One specialist answers each question. -members: [support.devices, support.accounts, support.fsd, support.general] -routing: - fallback: support.general ---- - -Ask one question per post. diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/mailboxes/lounge/MAILBOX.md b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/mailboxes/lounge/MAILBOX.md deleted file mode 100644 index 6fbb99816a..0000000000 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/mailboxes/lounge/MAILBOX.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -description: Where the support team chats. -members: [support.devices, support.accounts, support.fsd, support.general] ---- - -Say hello. diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/workers/help/WORKER.md b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/workers/help/WORKER.md new file mode 100644 index 0000000000..89f38e30c2 --- /dev/null +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/workers/help/WORKER.md @@ -0,0 +1,9 @@ +--- +description: Ask support anything. One specialist answers each question. +flow: coordinator +delegates: [support.devices, support.accounts, support.fsd, support.general] +routing: best-fit +fallback: support.general +--- + +Ask one question per post. diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/workers/lounge/WORKER.md b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/workers/lounge/WORKER.md new file mode 100644 index 0000000000..e26bf86fbb --- /dev/null +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/workers/lounge/WORKER.md @@ -0,0 +1,8 @@ +--- +description: Where the support team chats. +flow: coordinator +delegates: [support.devices, support.accounts, support.fsd, support.general] +routing: everyone +--- + +Say hello. diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md index 847bd4117a..b12ad0eb07 100644 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md @@ -1,39 +1,44 @@ # workforce-mailboxes › a routed post gets one answer -**Issue:** FIX-1610 (VG of the spec's PLAN; feeds the epic FIX-1592's closure, FIX-1601) +**Issue:** FIX-1610 (VG of the spec's PLAN; feeds the epic FIX-1592's closure, FIX-1601). Converted by FIX-1792 (BR-8, BR-9): the two mailboxes are coordinators' worker files now. -**Outcome:** A person's post to a mailbox that declares `routing:` runs exactly one specialist, the one its purpose and the recent lines point to. That specialist answers with the recent lines in view, and its answer lands in the mailbox as its own line every time, whatever the model does with its tools. A mailbox without the line behaves as before. The host writes no router: it passes `routeByPurpose(seats, { model })`. +**Outcome:** A person's post to a coordinator on `routing: best-fit` runs exactly one specialist, the one its purpose points to, and that specialist's answer lands in the person's conversation as its own line every time. A post sent while that specialist has not answered yet goes to it too, with no model call. A coordinator on `routing: everyone` hands each post to every delegate. The host writes no router: it registers Workforce's `coordinator` flow and names the model best fit's one call runs on. -**Input:** `fixtures/workforce/`: one team, four agent specialists (no `flow:`, each with a `description:`), one routed mailbox (`routing: fallback:` naming one of them), and one unrouted mailbox naming all four. `host.mts` is the app: the tree, the published packages, `wakeMemberSeats(seats)` and `routeByPurpose(seats, { model })`. Held-out: the specialists and the fallback are read off the tree (the routed mailbox's `members:` and `fallback:`), never hardcoded, and each run's posts carry fresh tokens. Renaming the team, both mailboxes and every worker, and reordering `members:`, passed unchanged. +**Converted (FIX-1792).** The folder name is kept so its verdict history stays in one place. Until FIX-1792 the routed mailbox was a `MAILBOX.md` with `routing: fallback:`, routed by `routeByPurpose`, and the unrouted one woke every member through `wakeMemberSeats`. They are coordinators now, key by key: `members:` became `delegates:`, `routing: { fallback: x }` became `routing: best-fit` and `fallback: x`, and the file with no `routing:` line got `routing: everyone`. The **unrouted** leg is now **everyone**: on a coordinator each delegate's answer lands in the conversation, where the unrouted mailbox kept none. -**Signal:** the host served in-process on in-memory stores and scripted models. One person posts eight times to the routed mailbox and once to the unrouted one. Everything graded is read back through the host's own router (each seat's conversations, dispatch runs included, and the mailbox's `mailbox-post` lines), plus the route model's own calls. +**Retired legs.** Two legs, and the two controls that served only them, have no subject on the coordinator flow, and are retired here under FIX-1792 BR-24 with the run that shows it (logged below): -- **import**: `routeByPurpose` resolves from `@flow-state-dev/workforce`. Checked first; the rest needs it. -- **source**: `host.mts` imports only `@flow-state-dev/*`, calls `routeByPurpose`, and builds no `dispatcher`, `keyedRouter` or `router`; exactly one mailbox file declares `routing:`. -- **one**: a device, an account and a framework post are each heard by their specialist and by no other member. -- **lands**: each of those, and the unclear post, has exactly one answer line in the mailbox, by the seat that was routed it. The answer script never calls the post tool, so a line there is the landing's. -- **followup**: a follow-up after the account answer is heard by the account specialist alone, routed by one evaluation whose lines included that answer. -- **held**: a post sent while the device specialist has not answered its last post (its answer to that one was empty, so no line) is heard by it alone, with no evaluation call. -- **context**: "where can I buy it?", routed to the fallback specialist, which was never sent the post naming "it", is answered by naming that post's item. The fallback's stored conversation keeps neither that post nor any message carrying another post's line. +- **followup** (a follow-up after an answer is routed by an evaluation that saw that answer's line) and its control `no-transcript`. The coordinator's best fit routes a post on its own words: its one evaluation is handed the post, not the conversation's recent lines. A follow-up sent while the specialist is still on the last post is still held for it (**held**); one sent after the answer is placed on its words alone. Live, "It sees it. It fails right after the password." went to `support.accounts`, where the mailbox sent it to `support.devices`. +- **context** ("where can I buy it?" answered by a specialist never sent the post naming "it", from that post's line) and its control `no-context`. A delegate's turn is handed the post it is delivered, and its own earlier deliveries from this conversation; never another delegate's lines. + +No goal proves either outcome on `main` after the conversion: the coordinator flow (FIX-1791) does not offer it. Whether it should is put to the product owner on FIX-1792's P1 PR. + +**Input:** `fixtures/workforce/`: one team, four agent specialists (no `flow:`, each with a `description:`), one coordinator on `routing: best-fit` with a `fallback:` naming one of them, and one on `routing: everyone` naming all four. `host.mts` is the app: the tree, the published packages, and the `coordinator` flow registered with the built-in `agent` as the flow its delegates take posts on. Held-out: the specialists and the fallback are read off the tree (the routed coordinator's `delegates:` and `fallback:`), never hardcoded, and each run's posts carry fresh tokens. + +**Signal:** the host served in-process on in-memory stores and scripted models. One person opens a conversation with each coordinator the way a page does (a session on `coordinator` naming it), posts six times to the routed one and once to the other. Everything graded is read back through the host's own router (each delegate's conversations, dispatch runs included, and the answer lines of the person's conversations), plus the route model's own calls. + +- **import**: `defineCoordinatorFlow` resolves from `@flow-state-dev/workforce`. Checked first; the rest needs it. +- **source**: `host.mts` imports only `@flow-state-dev/*`, calls `defineCoordinatorFlow`, and builds no `dispatcher`, `keyedRouter` or `router`; exactly one coordinator in the tree declares `routing: best-fit`. +- **one**: a device, an account and a framework post are each heard by their specialist and by no other delegate. +- **lands**: each of those, and the unclear post, has exactly one answer line in the conversation, by the specialist that was routed it. +- **held**: a post sent while the device specialist has not answered its last post (its turn on that one replied with nothing, so no line) is heard by it alone, with no evaluation call. - **fallback**: a post the evaluation cannot place is heard by the fallback alone. -- **unrouted**: the unrouted mailbox's post is heard once by each of the four agents, and nothing lands there. +- **everyone**: the post to the coordinator on `routing: everyone` is heard once by each of the four agents, and each one's answer lands in that conversation once. -**Anti-game:** no assertion on a route record, a dispatch handle, a router decision, or a unit test. Who ran is read from each seat's own kept conversation; what landed is read from the mailbox's lines; "no evaluation" and "the evaluation saw the line" are read from the route model's own calls, a real side effect. The scripts answer only from what they are handed: the route from the lines and post the route read (`[route:]`, else `[follow-up]` to whoever last spoke in those lines, else a failed call), the answer from its turn and the context it was shown. Waiting is polling until every request in the host settles, ungraded. +**Anti-game:** no assertion on a routing record, a dispatch handle, a router decision, or a unit test. Who ran is read from each delegate's own kept conversation; what landed is read from the person's conversation; "no evaluation" is read from the route model's own calls, a real side effect. The scripts answer only from what they are handed: the route from the post (`[route:]`, else a failed call), the answer from its turn. Waiting is polling until every request in the host settles, ungraded. **Model:** scripted, keyless (epic FIX-1592 D3), for every graded leg. The live leg (below) runs once with a key: the route on the host's own model string `vercel/typesafe-ai/jev` through the app's resolver, the answers on `vercel/openai/gpt-5.4-mini`. **Run:** `pnpm --dir goals exec tsx workforce-mailboxes/a-routed-post-gets-one-answer/run.mts` -**Live:** `GOAL_LIVE=1` on the same command, with `AI_GATEWAY_API_KEY`. After the scripted legs, the same host on real models: the POC's five posts, the laptop post its follow-up answers, and "where can I buy it?". Each post must reach one member (the expected one where the POC named it) and get one line by it. +**Live:** `GOAL_LIVE=1` on the same command, with `AI_GATEWAY_API_KEY`. After the scripted legs, the same host on real models: the POC's five posts, the laptop post's follow-up, and "where can I buy it?". Each post must reach one delegate (the expected one where the POC named it) and get one line by it. The follow-up names no expected delegate since the retirement above. **Controls:** on the same command. Each is applied by the check around the host, never inside a package. -- `GOAL_CONTROL=no-route`: the `routing:` line stripped from the mailbox file. Every agent hears every post and nothing lands. Must FAIL at **one**, **lands**, **followup**, **held**, **context** and **fallback**, and at nothing else. -- `GOAL_CONTROL=no-landing`: the agent kind replaced by one of the app's own that hears posts, answers with the recent lines in view, and posts nothing. Must FAIL at **lands**, and, because no specialist ever has a line so the hold keeps each next post on the last one, at **one**, **followup**, **held** and **context**. Nothing else. -- `GOAL_CONTROL=no-transcript`: the route's evaluation is handed the post without the recent lines. Must FAIL at **followup**, and at nothing else. -- `GOAL_CONTROL=no-context`: the host's notify strips the recent lines from each delivery. Must FAIL at **context**, and at nothing else. +- `GOAL_CONTROL=no-route`: the best-fit coordinator's file read as `routing: everyone`. Every delegate hears every post and each answers it. Must FAIL at **one**, **lands**, **held** and **fallback**, and at nothing else. +- `GOAL_CONTROL=no-landing`: the `agent` flow replaced by one of the app's own that takes a delegated post, answers it in its own conversation, and hands nothing back. Must FAIL at **lands**; because no specialist ever answers, the hold keeps each next post on the first specialist, so at **one** and **fallback** too; and at **everyone**, whose answers never land. Nothing else. -The **import**, **source** and **unrouted** legs have no control; each was reddened by hand, logged below. +The **import** and **source** legs have no control; each was reddened by hand, logged below. ## Verdict log | Date | Commit | Model | Verdict | Notes | diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts index 7a0e3e9d2f..41f09f9547 100644 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts @@ -1,32 +1,30 @@ /** - * A fresh app with a routed support mailbox, as a new app would write it. + * A fresh app with a routed support coordinator, as a new app would write it. * * Everything here comes from the published packages and the team's files: no - * kitchen-sink code, and no dispatcher or router of the app's own. The wake is - * `wakeMemberSeats(copies, { installation })` in the built-in mailbox kind's - * notify slot, and the route is `routeByPurpose(copies, { model, installation })`. - * Every worker flow is registered once, and a woken worker's conversation is a - * session on its flow, naming it. A mailbox that declares - * `routing:` in its file is routed; one that doesn't wakes every agent member. - * The goal check reads this file's source to hold it to that. + * kitchen-sink code, and no dispatcher or router of the app's own. The + * coordinators are worker files (`flow: coordinator`); the routed one says + * `routing: best-fit` and names its `fallback:`. The app registers + * Workforce's `coordinator` flow once, with the built-in `agent` as the flow + * its delegates take posts on, and names the model best fit's one evaluator + * call runs on. Every worker flow is registered once, and a delegate's + * conversation is a session on its flow, naming it. The goal check reads this + * file's source to hold it to that. * - * Keyless by default: the seats answer from a scripted model, and the route's - * one evaluation is scripted by its block name, `mailbox-route`. The scripts - * are the app's test doubles, and they answer from what they are handed: + * Keyless by default: the delegates answer from a scripted model, and best + * fit's one evaluation is scripted by its block name, `coordinator-route`. + * The scripts are the app's test doubles, and they answer from what they are + * handed: * - * - the route: a post naming `[route:]` goes to that member; a post - * marked `[follow-up]` goes to whoever last spoke in the lines the route - * read; anything else fails the call, so the mailbox's fallback takes it. - * - an answer: `[answer:none]` replies with nothing; `[answer:from-context]` - * names the `item-…` it finds in the recent lines it was shown; anything - * else acknowledges the post's `tok-…`. Every reply carries - * {@link REPLY_MARKER}. No answer calls the post tool. + * - the route: a post naming `[route:]` goes to that delegate; + * anything else fails the call, so the coordinator's fallback takes it. + * - an answer: `[answer:none]` replies with nothing, which is a turn with no + * answer; anything else acknowledges the post's `tok-…`. Every reply carries + * {@link REPLY_MARKER}. * * `seams` is the goal check's, for its controls and its live leg, and nothing * else. An app passes nothing. */ -import { createSessionClient } from "@flow-state-dev/client"; -import type { BlockDefinition } from "@flow-state-dev/core"; import type { EvaluationModel, FlowInstance, ModelResolver } from "@flow-state-dev/core/types"; import { createFlowState, inMemoryStores, type FlowState } from "@flow-state-dev/engine"; import { @@ -37,20 +35,17 @@ import { type MockGeneratorScriptStep } from "@flow-state-dev/testing"; import { - mailboxInstances, createWorkerInstallation, - defineMailboxFlow, + defineAgentWorkerFlow, + defineCoordinatorFlow, hireWorkforce, - openMailboxes, - routeByPurpose, - wakeMemberSeats, - type MailboxManifest, - type WorkerInstallation + type WorkerInstallation, + type WorkerManifest } from "@flow-state-dev/workforce"; -import { readMailboxesDirectory, readWorkforce } from "@flow-state-dev/workforce/loader"; +import { readWorkforce } from "@flow-state-dev/workforce/loader"; -/** The user the mailboxes are opened under, and who posts to them. */ -export const MAILBOX_OWNER = "u_routed_host"; +/** The user who talks to the coordinators. */ +export const OWNER = "u_routed_host"; /** What every scripted answer carries. */ export const REPLY_MARKER = "[reply:routed-host]"; @@ -60,15 +55,11 @@ export const ROUTE_MODEL = "vercel/typesafe-ai/jev"; /** The goal check's seams. An app passes none. */ export interface HostSeams { - /** The mailbox files as the host reads them, before anything is built. */ - adaptMailboxes?: (mailboxes: MailboxManifest[]) => MailboxManifest[]; - /** Worker flows the app registers beside the built-ins, by name, built on the installation. */ - kinds?: (installation: WorkerInstallation) => Record; - /** The wake the mailbox's notify slot runs. */ - adaptNotify?: (wake: BlockDefinition) => BlockDefinition; - /** The route's scripted evaluation model, as the resolver hands it over. */ - adaptEvaluation?: (model: EvaluationModel) => EvaluationModel; - /** Real models: this resolver resolves the route's model and the seats' answers. */ + /** The worker files as the host reads them, before anything is built. */ + adaptWorkers?: (workers: WorkerManifest[]) => WorkerManifest[]; + /** The `agent` flow, replaced by one of the app's own built on the installation. */ + agent?: (installation: WorkerInstallation) => { kind: string }; + /** Real models: this resolver resolves the route's model and the delegates' answers. */ live?: { modelResolver: ModelResolver }; } @@ -76,8 +67,7 @@ export interface HostSeams { export interface RoutedHost { state: FlowState; router: Awaited>; - seats: FlowInstance[]; - mailboxes: MailboxManifest[]; + copies: FlowInstance[]; /** Every call the scripted route evaluation received. Empty on the live leg. */ routeCalls: MockEvaluationCall[]; } @@ -88,26 +78,19 @@ function textOf(content: unknown): string { return Array.isArray(content) ? content.map((part) => (part as { text?: string }).text ?? "").join("") : ""; } -/** The route's scripted evaluation. Reads the `{ recent, post }` state the route hands it. */ +/** Best fit's scripted evaluation. Reads the `{ post }` state the coordinator hands it. */ function scriptedRoute() { return mockEvaluationModel({ answers: ({ state }) => { - const { recent = [], post } = state as { - recent?: Array<{ from: string; text: string }>; - post: { from: string; text: string }; - }; + const { post } = state as { post: { from: string; text: string } }; const named = /\[route:([^\]\s]+)\]/.exec(post.text)?.[1]; if (named !== undefined) return { member: { type: "choice", choice: named } }; - if (post.text.includes("[follow-up]")) { - const last = [...recent].reverse().find((line) => line.from !== post.from); - if (last !== undefined) return { member: { type: "choice", choice: last.from } }; - } - throw new Error("the scripted route has no member for this post"); + throw new Error("the scripted route has no delegate for this post"); } }); } -/** The seats' scripted answer. Never calls a tool. */ +/** The delegates' scripted answer. Never calls a tool. */ function scriptedAnswer(): MockGeneratorInstance { return { name: "agent-answer", @@ -116,80 +99,49 @@ function scriptedAnswer(): MockGeneratorInstance { next: (input: unknown): MockGeneratorScriptStep => { const messages = input as Array<{ role: string; content: unknown }>; const turn = textOf([...messages].reverse().find((m) => m.role === "user")?.content); - const system = messages.filter((m) => m.role === "system").map((m) => textOf(m.content)).join("\n"); if (turn.includes("[answer:none]")) return { text: "" }; const token = /tok-[a-z0-9]+/.exec(turn)?.[0] ?? ""; - if (turn.includes("[answer:from-context]")) { - const item = /item-[a-z0-9]+/.exec(system)?.[0]; - return { text: `${REPLY_MARKER} ${token} ${item === undefined ? "Buy what?" : `You can buy the ${item} at the shop.`}` }; - } return { text: `${REPLY_MARKER} ${token} noted.` }; } }; } /** - * Read the team's files, hire its seats, build its mailboxes with the wake and - * the route, and open them. + * Read the team's files, register one copy of each worker flow, the + * coordinator's among them, and serve them. * * @param tree The workforce root to read. * @param seams The goal check's seams. An app passes nothing. */ export async function startRoutedHost(tree: string, seams: HostSeams = {}): Promise { - const { workers, errors } = await readWorkforce(tree); - const read = await readMailboxesDirectory(tree); - if (errors.length > 0 || read.errors.length > 0) { - throw new Error(`the tree did not load: ${[...errors, ...read.errors].map((e) => `${e.path}: ${String(e.error)}`).join(", ")}`); + const read = await readWorkforce(tree); + if (read.errors.length > 0) { + throw new Error(`the tree did not load: ${read.errors.map((e) => `${e.path}: ${String(e.error)}`).join(", ")}`); } - const mailboxes = seams.adaptMailboxes?.(read.mailboxes) ?? read.mailboxes; + const workers = seams.adaptWorkers?.(read.workers) ?? read.workers; - // The flows first: the wake and the route reach the workers on these copies, - // never a mailbox's stored members. + // The flows are handed to the installation lazily, so each can be built on it. let flows: Record = {}; const installation = createWorkerInstallation({ standardWorkers: workers, workerFlows: () => flows as never }); - flows = seams.kinds?.(installation) ?? {}; - const seats = hireWorkforce(installation); - const wake = wakeMemberSeats(seats, { installation }); - const mailboxFlows = mailboxInstances(mailboxes, { - kinds: { - mailbox: defineMailboxFlow({ - notify: seams.adaptNotify?.(wake) ?? wake, - route: routeByPurpose(seats, { model: ROUTE_MODEL, installation }) - }) - } - }); + const agent = seams.agent?.(installation) ?? defineAgentWorkerFlow({ installation }); + flows = { + agent, + coordinator: defineCoordinatorFlow({ installation, delegateFlows: [agent as never], routeModel: ROUTE_MODEL }) + }; + const copies = hireWorkforce(installation); const route = scriptedRoute(); const state = createFlowState({ - flows: { - ...Object.fromEntries(mailboxFlows.map((flow) => [flow.kind, flow])), - ...Object.fromEntries(seats.map((seat) => [seat.id, seat])) - }, + flows: Object.fromEntries(copies.map((copy) => [copy.id, copy])), stores: { default: { primary: inMemoryStores() } }, modelResolver: seams.live?.modelResolver ?? createMockModelResolver({ generators: { "agent-answer": scriptedAnswer() }, - evaluators: { "mailbox-route": seams.adaptEvaluation?.(route) ?? route }, + evaluators: { "coordinator-route": route as EvaluationModel }, policy: "allow" }) } as never); const router = await state.getRouter(); - - // The session client over the app's own router: the app is the server. - const sessions = createSessionClient({ - fetcher: async (input, init) => { - const url = new URL(String(input), "http://routed-host.local"); - const path = url.pathname - .replace(/^\/api\/flows\/?/, "") - .split("/") - .filter((segment) => segment.length > 0) - .map(decodeURIComponent); - const method = (init?.method ?? "GET").toUpperCase() as "GET" | "POST" | "PATCH" | "DELETE"; - return await router[method](new Request(url, init), { params: { path } }); - } - }); - await openMailboxes(mailboxes, { client: sessions, userId: MAILBOX_OWNER }); - - return { state, router, seats, mailboxes, routeCalls: seams.live === undefined ? route.calls : [] }; + return { state, router, copies, routeCalls: seams.live === undefined ? route.calls : [] }; } diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts index d77581f52f..08808d57bf 100644 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts @@ -1,43 +1,38 @@ /** - * Goal check: a person's post to a mailbox that declares `routing:` runs - * exactly one specialist, the one its purpose and the recent lines point to; - * that specialist answers with the recent lines in view, and its answer lands - * in the mailbox as its own line whatever the model does with its tools. A - * mailbox without the line behaves as before. + * Goal check: a person's post to a coordinator on `routing: best-fit` runs + * exactly one specialist, the one its purpose points to, and that + * specialist's answer lands in the person's conversation as its own line + * every time. A coordinator on `routing: everyone` hands each post to every + * delegate. * * Real path, scripted models, out of CI. See goal.md for the contract. * * The host is `host.mts`, an app built from the published packages and the * fixture tree alone. It is served in-process, and everything graded is read - * back through its own router (each seat's conversations, dispatch runs - * included, and the mailbox's lines), plus the route model's own calls. Legs: + * back through its own router (each delegate's conversations, dispatch runs + * included, and the person's conversation's answer lines), plus the route + * model's own calls. Legs: * - * import `routeByPurpose` resolves from `@flow-state-dev/workforce`. - * source the host imports only `@flow-state-dev/*`, calls - * `routeByPurpose`, and builds no dispatcher or router; the - * routed mailbox's file declares `routing:`. + * import `defineCoordinatorFlow` resolves from `@flow-state-dev/workforce`. + * source the host imports only `@flow-state-dev/*`, registers the + * coordinator flow, and builds no dispatcher or router; exactly + * one coordinator in the tree routes by best fit. * one a device, an account and a framework post each reach their * specialist, and nobody else. * lands each of those, and the unclear post, gets exactly one line in - * the mailbox, by the seat that answered it. - * followup a follow-up after the account answer reaches the account - * specialist, by an evaluation that saw that answer's line. + * the conversation, by the specialist that answered it. * held a post sent while the specialist has not answered yet goes to * it alone, with no evaluation. - * context "where can I buy it?" is answered, by a specialist that was - * never sent the post naming "it", from that post's line; its - * stored conversation keeps only its own posts and answers. * fallback a post the evaluation cannot place goes to the fallback alone. - * unrouted a post to the mailbox with no `routing:` wakes every agent. + * everyone a post to the coordinator on `routing: everyone` reaches every + * agent once, and each answer lands there once. * * With GOAL_LIVE=1 a live leg runs after them, on real models: the route on * the host's own evaluation model string, the answers on a small chat model. * * Run: pnpm --dir goals exec tsx workforce-mailboxes/a-routed-post-gets-one-answer/run.mts - * Controls: GOAL_CONTROL=no-route (the routing line stripped from the file) - * GOAL_CONTROL=no-landing (the agent kind replaced by one that hears posts and never lands) - * GOAL_CONTROL=no-transcript (the recent lines hidden from the route's evaluation) - * GOAL_CONTROL=no-context (the recent lines stripped from the routed delivery) + * Controls: GOAL_CONTROL=no-route (the best-fit coordinator read as `routing: everyone`) + * GOAL_CONTROL=no-landing (the agent flow replaced by one that answers a delegated post and hands nothing back) * Live: GOAL_LIVE=1 (needs AI_GATEWAY_API_KEY) */ import { randomUUID } from "node:crypto"; @@ -45,17 +40,17 @@ import { readFileSync } from "node:fs"; import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; import { z } from "zod"; -import { createModelResolver, defineFlow, generator, handler, type BlockDefinition } from "@flow-state-dev/core"; -import type { EvaluationModel, ModelResolver } from "@flow-state-dev/core/types"; +import { createModelResolver, defineFlow, generator, handler } from "@flow-state-dev/core"; +import type { ModelResolver } from "@flow-state-dev/core/types"; import { - mailboxNotifyInputSchema, + delegatedPostSchema, workerConfigOf, workerConfigSchema, - type MailboxManifest, - type MailboxNotifyInput, - type WorkerInstallation + type DelegatedPost, + type WorkerInstallation, + type WorkerManifest } from "@flow-state-dev/workforce"; -import { readMailboxesDirectory } from "@flow-state-dev/workforce/loader"; +import { readWorkforce } from "@flow-state-dev/workforce/loader"; import { gatewayModel, runGoal } from "../../lib/index.mts"; import type { HostSeams, RoutedHost } from "./host.mts"; @@ -66,40 +61,29 @@ const LIVE = process.env.GOAL_LIVE === "1"; /** The legs each control must redden, and only those. Set from the runs logged in goal.md. */ const EXPECTED: Record = { - // Unrouted, every agent hears every post and nothing lands: every routed leg. - "no-route": ["one", "lands", "followup", "held", "context", "fallback"], - // No line ever lands, so a specialist is always "still on" the last post and - // the hold keeps the next one: lands, and every leg a hold then diverts. - "no-landing": ["one", "lands", "followup", "held", "context"], - // The evaluation cannot see the account answer, so the follow-up falls back. - "no-transcript": ["followup"], - // The routed turn gets no lines, so "it" has no referent. - "no-context": ["context"] + // Every delegate hears every post, and each answers it: every best-fit leg. + "no-route": ["one", "lands", "held", "fallback"], + // No answer ever lands, so the specialist on the first post holds every next + // one: lands, every leg the hold then diverts, and the answers on everyone. + "no-landing": ["one", "lands", "fallback", "everyone"] }; if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); } -/** A seat's turn as a routed or unrouted post is heard: ` in : `. */ -const heard = (post: MailboxNotifyInput) => `${post.author ?? post.principal} in ${post.mailboxId}: ${post.body}`; +/** How a delegate's turn reads a delegated post: `, through : `. */ +const heard = (post: DelegatedPost) => `${post.from}, through ${post.coordinator}: ${post.body}`; /** - * `no-landing`'s kind: hears posts and answers through `agent-answer`, with - * the recent lines as context, and posts nothing. A kind of the app's own gets - * the routed mark and decides what to do with it; this one does nothing. + * `no-landing`'s agent flow: takes a delegated post and answers it through + * `agent-answer` in its own conversation, and never hands the answer back. */ -function quietAgentKind(installation: WorkerInstallation) { +function quietAgentFlow(installation: WorkerInstallation) { const answer = generator({ name: "agent-answer", - inputSchema: mailboxNotifyInputSchema, + inputSchema: delegatedPostSchema, model: "scripted/answer", - prompt: (_post: MailboxNotifyInput, ctx) => (workerConfigOf(ctx) as { instructions?: string }).instructions ?? "", - context: [ - (post: MailboxNotifyInput) => - post.recent === undefined || post.recent.length === 0 - ? undefined - : ["Recent lines in the mailbox, oldest first:", ...post.recent.map((l) => `- ${l.author ?? l.principal}: ${l.body}`)].join("\n") - ], + prompt: (_post: DelegatedPost, ctx) => (workerConfigOf(ctx) as { instructions?: string }).instructions ?? "", user: heard }); const run = generator({ name: "agent-answer", inputSchema: z.object({ message: z.string() }), model: "scripted/answer", prompt: "", user: (i: { message: string }) => i.message }); @@ -111,13 +95,12 @@ function quietAgentKind(installation: WorkerInstallation) { }); return defineFlow({ kind: "agent", - cardinality: "collection", configSchema: workerConfigSchema(), session: installation.session(), resources: { ...installation.resources }, request: { onStarted: loadWorker }, actions: { run: { inputSchema: z.object({ message: z.string() }), block: run, userMessage: (i: { message: string }) => i.message } }, - internal: { actions: { onMailboxPost: { inputSchema: mailboxNotifyInputSchema, block: answer, userMessage: heard } } } + internal: { actions: { onDelegatedPost: { inputSchema: delegatedPostSchema, block: answer, userMessage: heard } } } } as never); } @@ -126,34 +109,18 @@ function controlSeams(): HostSeams { switch (CONTROL) { case "no-route": return { - adaptMailboxes: (mailboxes: MailboxManifest[]) => - mailboxes.map(({ declared: { routing: _routing, ...declared }, ...rest }) => ({ ...rest, declared })) + adaptWorkers: (workers: WorkerManifest[]) => + workers.map((w) => (w.declared.routing === "best-fit" ? { ...w, declared: { ...w.declared, routing: "everyone" } } : w)) }; case "no-landing": - return { kinds: (installation: WorkerInstallation) => ({ agent: quietAgentKind(installation) }) }; - case "no-transcript": - return { - adaptEvaluation: (model: EvaluationModel) => - ({ - ...model, - doEvaluate: (call: { state: { recent?: unknown } }) => { - const { recent: _recent, ...state } = call.state; - return (model as unknown as { doEvaluate: (c: unknown) => unknown }).doEvaluate({ ...call, state }); - } - }) as unknown as EvaluationModel - }; - case "no-context": - return { - adaptNotify: (wake: BlockDefinition) => - wake.connectInput(({ recent: _recent, ...post }: MailboxNotifyInput) => post) - }; + return { agent: (installation: WorkerInstallation) => quietAgentFlow(installation) }; default: return {}; } } type Message = { role: string; text: string }; -type Line = { author?: string; body: string }; +type Line = { author?: string; text: string }; /** Reads through the host's own router, the routes a page reads. */ function reader(app: RoutedHost, owner: string) { @@ -169,16 +136,22 @@ function reader(app: RoutedHost, owner: string) { return { status: res.status, text: await res.text() }; }; return { - post: async (mailboxId: string, body: string) => { - const res = await call("POST", ["mailbox", "actions", "post"], { userId: owner, sessionId: mailboxId, input: { body } }); - if (res.status !== 200) throw new Error(`post "${body}" refused: ${res.status} ${res.text.slice(0, 300)}`); + /** The person's conversation with a coordinator, opened the way a page opens one. */ + open: async (coordinator: string) => { + const res = await call("POST", ["coordinator", "sessions"], { userId: owner, state: { workerId: coordinator } }); + if (res.status !== 201) throw new Error(`a conversation with ${coordinator} was not opened: ${res.status} ${res.text.slice(0, 300)}`); + return (JSON.parse(res.text) as { session: { id: string } }).session.id; }, - /** A seat's conversations of one mailbox: the kept messages of each. */ - conversationsOf: async (seat: string, mailboxId: string): Promise => { + post: async (conversation: string, message: string) => { + const res = await call("POST", ["coordinator", "actions", "run"], { userId: owner, sessionId: conversation, input: { message } }); + if (res.status !== 200) throw new Error(`post "${message}" refused: ${res.status} ${res.text.slice(0, 300)}`); + }, + /** A delegate's conversations under one of the person's: the kept messages of each. */ + conversationsOf: async (seat: string, conversation: string): Promise => { const listed = await call("GET", ["sessions"], undefined, `?flowId=agent&state.workerId=${encodeURIComponent(seat)}&userId=${owner}&include=dispatch-runs&limit=100`); const rows = (JSON.parse(listed.text) as { sessions?: Array<{ id: string; parentSessionId?: string | null }> }).sessions ?? []; const out: Message[][] = []; - for (const row of rows.filter((r) => r.parentSessionId === mailboxId)) { + for (const row of rows.filter((r) => r.parentSessionId === conversation)) { const state = await call("GET", ["sessions", row.id, "state"], undefined, "?include_items=true&item_types=message&limit=1000"); const items = (JSON.parse(state.text) as { items?: Array<{ role?: string; transient?: boolean; content?: Array<{ text?: string }> }> }).items ?? []; out.push( @@ -189,16 +162,21 @@ function reader(app: RoutedHost, owner: string) { } return out; }, - /** The mailbox's lines, as its `mailbox-post` items carry them, oldest first. */ - linesOf: async (mailboxId: string): Promise => { - const state = await call("GET", ["sessions", mailboxId, "state"], undefined, "?include_items=true&item_types=component&limit=1000"); - const items = (JSON.parse(state.text) as { items?: Array<{ component?: string; data?: Line }> }).items ?? []; - return items.filter((item) => item.component === "mailbox-post").map((item) => item.data!); + /** The conversation's lines, oldest first: the person's posts, and each answer under its delegate's name. */ + linesOf: async (conversation: string): Promise => { + const state = await call("GET", ["sessions", conversation, "state"], undefined, "?include_items=true&item_types=message&limit=1000"); + const items = (JSON.parse(state.text) as { items?: Array<{ role?: string; agentName?: string; transient?: boolean; content?: Array<{ text?: string }> }> }).items ?? []; + return items + .filter((item) => item.transient !== true && (item.role === "user" || item.agentName !== undefined)) + .map((item) => ({ + ...(item.agentName === undefined ? {} : { author: item.agentName }), + text: (item.content ?? []).map((c) => c.text ?? "").join("") + })); } }; } -/** Until every request in the host has settled, twice over (a landing starts one more). Not graded. */ +/** Until every request in the host has settled, three reads running. Not graded. */ async function quiet(app: RoutedHost, ms: number): Promise { const runtime = await app.state.getRuntime(); const busy = async () => { @@ -215,133 +193,115 @@ async function quiet(app: RoutedHost, ms: number): Promise { } } +/** The two coordinators the fixture holds: the routed one and the one on `everyone`, read off the tree. */ +async function coordinatorsOf(tree: string) { + const { workers } = await readWorkforce(tree); + const coordinators = workers.filter((w) => w.declared.flow === "coordinator"); + const help = coordinators.find((w) => w.declared.routing === "best-fit"); + const lounge = coordinators.find((w) => w.declared.routing === "everyone"); + if (help === undefined || lounge === undefined) throw new Error("the fixture needs one coordinator on best-fit and one on everyone"); + const delegates = help.declared.delegates as string[]; + const fallback = help.declared.fallback as string; + return { help, lounge, delegates, fallback }; +} + /** The scripted legs. */ async function scriptedLegs(fail: (leg: string, line: string) => void, evidence: string[]): Promise { - const { mailboxes } = await readMailboxesDirectory(TREE); - const help = mailboxes.find((c) => c.declared.routing !== undefined); - const lounge = mailboxes.find((c) => c.declared.routing === undefined); - if (help === undefined || lounge === undefined) throw new Error("the fixture needs one routed and one unrouted mailbox"); - const members = help.declared.members as string[]; - const fallback = (help.declared.routing as { fallback: string }).fallback; - const [s1, s2, s3] = members.filter((m) => m !== fallback); - if (s3 === undefined) throw new Error(`the routed mailbox needs three specialists beside its fallback; read ${members.join(", ")}`); + const { help, lounge, delegates, fallback } = await coordinatorsOf(TREE); + const [s1, s2, s3] = delegates.filter((m) => m !== fallback); + if (s3 === undefined) throw new Error(`the routed coordinator needs three specialists beside its fallback; read ${delegates.join(", ")}`); - const { startRoutedHost, MAILBOX_OWNER, REPLY_MARKER } = await import("./host.mts"); + const { startRoutedHost, OWNER, REPLY_MARKER } = await import("./host.mts"); const app = await startRoutedHost(TREE, controlSeams()); - const io = reader(app, MAILBOX_OWNER); + const io = reader(app, OWNER); const run = randomUUID().replace(/-/g, "").slice(0, 10); const tok = (n: number) => `tok-${run}p${n}`; - const item = `item-${run}`; const posts = { device: `[route:${s1}] ${tok(1)} My phone stopped charging, and it's a brand new cable.`, account: `[route:${s2}] ${tok(2)} Different thing: I was charged twice for my subscription this month.`, - followup: `[follow-up] ${tok(3)} It was the visa card.`, framework: `[route:${s3}] ${tok(4)} How do I make a generator return structured output?`, unclear: `${tok(5)} Who do I ask about getting a parking pass?`, - unanswered: `[route:${s1}] [answer:none] ${tok(6)} My laptop ${item} won't join the office wifi.`, + unanswered: `[route:${s1}] [answer:none] ${tok(6)} My laptop won't join the office wifi.`, held: `${tok(7)} It sees the network. It fails right after the password.`, - buyIt: `[route:${fallback}] [answer:from-context] ${tok(8)} Where can I buy it?`, lounge: `${tok(9)} Morning, all.` }; - const heardBy = async (mailboxId: string, token: string) => { - const out: string[] = []; - for (const seat of members) { - const turns = (await io.conversationsOf(seat, mailboxId)).flat().filter((m) => m.role === "user" && m.text.includes(token)); - if (turns.length > 0) out.push(seat); - } - return out; - }; - const callsFor = (token: string) => - app.routeCalls.filter((c) => (c.state as { post: { text: string } }).post.text.includes(token)); - try { - for (const body of [posts.device, posts.account, posts.followup, posts.framework, posts.unclear, posts.unanswered, posts.held, posts.buyIt]) { - await io.post(help.id, body); + const conversation = await io.open(help.id); + const loungeConversation = await io.open(lounge.id); + const heardBy = async (token: string) => { + const out: string[] = []; + for (const seat of delegates) { + const turns = (await io.conversationsOf(seat, conversation)).flat().filter((m) => m.role === "user" && m.text.includes(token)); + if (turns.length > 0) out.push(seat); + } + return out; + }; + const callsFor = (token: string) => + app.routeCalls.filter((c) => (c.state as { post: { text: string } }).post.text.includes(token)); + + for (const body of [posts.device, posts.account, posts.framework, posts.unclear, posts.unanswered, posts.held]) { + await io.post(conversation, body); await quiet(app, 8_000); } - await io.post(lounge.id, posts.lounge); + await io.post(loungeConversation, posts.lounge); await quiet(app, 8_000); - const lines = await io.linesOf(help.id); + const lines = await io.linesOf(conversation); // ---- one: each post reaches its specialist, and nobody else ------------- for (const [token, want] of [[tok(1), s1], [tok(2), s2], [tok(4), s3]] as const) { - const who = await heardBy(help.id, token); + const who = await heardBy(token); if (who.length !== 1 || who[0] !== want) fail("one", `${token} was heard by [${who.join(", ")}] (want [${want}])`); else evidence.push(`${token} reached ${want} alone`); } - // ---- lands: one line each, by the seat that answered -------------------- + // ---- lands: one line each, by the specialist that answered -------------- for (const [token, want] of [[tok(1), s1], [tok(2), s2], [tok(4), s3], [tok(5), fallback]] as const) { - const answers = lines.filter((l) => l.author !== undefined && l.body.includes(token)); - if (answers.length !== 1 || answers[0]!.author !== want || !answers[0]!.body.includes(REPLY_MARKER)) { - fail("lands", `${token} has ${answers.length} answer line(s) in ${help.id}: ${JSON.stringify(answers)} (want one, by ${want})`); + const answers = lines.filter((l) => l.author !== undefined && l.text.includes(token)); + if (answers.length !== 1 || answers[0]!.author !== want || !answers[0]!.text.includes(REPLY_MARKER)) { + fail("lands", `${token} has ${answers.length} answer line(s) in ${help.id}'s conversation: ${JSON.stringify(answers)} (want one, by ${want})`); } else evidence.push(`${token}'s answer landed once, by ${want}`); } - // ---- followup: to the account specialist, by an evaluation that saw its line - { - const who = await heardBy(help.id, tok(3)); - const calls = callsFor(tok(3)); - const saw = calls.some((c) => ((c.state as { recent?: Array<{ from: string }> }).recent ?? []).some((l) => l.from === s2)); - if (who.length !== 1 || who[0] !== s2) fail("followup", `${tok(3)} was heard by [${who.join(", ")}] (want [${s2}])`); - if (calls.length !== 1 || !saw) fail("followup", `${tok(3)}'s route made ${calls.length} evaluation call(s), ${saw ? "seeing" : "not seeing"} a line by ${s2} (want one, seeing it)`); - if (who.length === 1 && who[0] === s2 && calls.length === 1 && saw) evidence.push(`the follow-up reached ${s2} by one evaluation that saw its line`); - } - // ---- held: sent before the specialist answered, so no evaluation -------- { - const unanswered = lines.filter((l) => l.author !== undefined && l.body.includes(tok(6))); - if (unanswered.length > 0) fail("held", `${tok(6)} was answered in the mailbox, so ${tok(7)} was not sent before an answer: ${JSON.stringify(unanswered)}`); - const who = await heardBy(help.id, tok(7)); + const unanswered = lines.filter((l) => l.author !== undefined && l.text.includes(tok(6))); + if (unanswered.length > 0) fail("held", `${tok(6)} was answered in the conversation, so ${tok(7)} was not sent before an answer: ${JSON.stringify(unanswered)}`); + const who = await heardBy(tok(7)); const calls = callsFor(tok(7)); if (who.length !== 1 || who[0] !== s1) fail("held", `${tok(7)} was heard by [${who.join(", ")}] (want [${s1}])`); if (calls.length !== 0) fail("held", `${tok(7)}'s route made ${calls.length} evaluation call(s) (want 0)`); if (unanswered.length === 0 && who.length === 1 && who[0] === s1 && calls.length === 0) evidence.push(`${tok(7)} went to ${s1}, still on the last post, with no evaluation`); } - // ---- context: answered from a line the answering seat was never sent ---- - { - let ok = true; - const answer = lines.filter((l) => l.author === fallback && l.body.includes(tok(8))); - if (answer.length !== 1 || !answer[0]!.body.includes(item)) { - ok = false; - fail("context", `${fallback}'s answer to ${tok(8)} does not name ${item}: ${JSON.stringify(answer)}`); - } - // The post that named "it", as the mailbox holds it: in no message the seat kept. - const named = lines.find((l) => l.author === undefined && l.body.includes(tok(6)))?.body ?? posts.unanswered; - const conversation = (await io.conversationsOf(fallback, help.id)).flat(); - if (conversation.some((m) => m.text.includes(named))) { - ok = false; - fail("context", `${fallback}'s stored conversation keeps the line it was shown as context`); - } - // Each kept message is about one post: its own. The lines it was shown are in none. - const crowded = conversation.filter((m) => new Set(m.text.match(/tok-[a-z0-9]+/g) ?? []).size > 1); - if (crowded.length > 0) { - ok = false; - fail("context", `${fallback} kept message(s) carrying other posts' lines: ${JSON.stringify(crowded)}`); - } - if (ok) evidence.push(`${fallback}, never sent ${tok(6)}, answered "${answer[0]!.body}", and kept none of the lines it was shown`); - } - // ---- fallback: a post nobody could place goes to the fallback alone ------ { - const who = await heardBy(help.id, tok(5)); + const who = await heardBy(tok(5)); if (who.length !== 1 || who[0] !== fallback) fail("fallback", `${tok(5)} was heard by [${who.join(", ")}] (want [${fallback}])`); else evidence.push(`${tok(5)} went to ${fallback} alone`); } - // ---- unrouted: every agent member hears it once; nothing lands ----------- + // ---- everyone: every agent hears it once; each answer lands once --------- { - const agents = lounge.declared.members as string[]; + const agents = lounge.declared.delegates as string[]; + let ok = true; + for (const seat of agents) { + const turns = (await io.conversationsOf(seat, loungeConversation)).flat().filter((m) => m.role === "user" && m.text.includes(tok(9))); + if (turns.length !== 1) { + ok = false; + fail("everyone", `${seat} heard ${tok(9)} in ${turns.length} turns (want 1)`); + } + } + const loungeLines = (await io.linesOf(loungeConversation)).filter((l) => l.author !== undefined && l.text.includes(tok(9))); for (const seat of agents) { - const turns = (await io.conversationsOf(seat, lounge.id)).flat().filter((m) => m.role === "user" && m.text.includes(tok(9))); - if (turns.length !== 1) fail("unrouted", `${seat} heard ${tok(9)} in ${turns.length} turns (want 1)`); + const by = loungeLines.filter((l) => l.author === seat); + if (by.length !== 1) { + ok = false; + fail("everyone", `${lounge.id}'s conversation holds ${by.length} answer line(s) by ${seat} to ${tok(9)} (want 1)`); + } } - const loungeLines = await io.linesOf(lounge.id); - if (loungeLines.length !== 1) fail("unrouted", `${lounge.id} holds ${loungeLines.length} lines (want the post alone)`); - evidence.push(`${lounge.id}: ${agents.length} agents woken`); + if (ok) evidence.push(`${lounge.id}: ${agents.length} agents heard the post once, and each answer landed once`); } } finally { await app.state.dispose(); @@ -367,18 +327,16 @@ async function liveLeg(fail: (leg: string, line: string) => void, evidence: stri intents: { chat: [answerModel] } }); - const { mailboxes } = await readMailboxesDirectory(TREE); - const help = mailboxes.find((c) => c.declared.routing !== undefined)!; - const members = help.declared.members as string[]; - const fallback = (help.declared.routing as { fallback: string }).fallback; - const [devices, accounts, framework] = members.filter((m) => m !== fallback); + const { help, delegates, fallback } = await coordinatorsOf(TREE); + const [devices, accounts, framework] = delegates.filter((m) => m !== fallback); - const { startRoutedHost, MAILBOX_OWNER } = await import("./host.mts"); + const { startRoutedHost, OWNER } = await import("./host.mts"); const app = await startRoutedHost(TREE, { live: { modelResolver } }); - const io = reader(app, MAILBOX_OWNER); + const io = reader(app, OWNER); const posts: Array<{ body: string; want?: string }> = [ { body: "Hi, my laptop won't join the office wifi since this morning.", want: devices }, - { body: "It sees it. It fails right after the password.", want: devices }, + // The follow-up names no subject; best fit routes it on its own words (goal.md, "Retired legs"). + { body: "It sees it. It fails right after the password." }, { body: "My phone stopped charging, and it's a brand new cable.", want: devices }, { body: "Where can I buy it?" }, { body: "Different thing: I was charged twice for my subscription this month.", want: accounts }, @@ -386,23 +344,24 @@ async function liveLeg(fail: (leg: string, line: string) => void, evidence: stri { body: "Who do I ask about getting a parking pass?", want: fallback } ]; try { + const conversation = await io.open(help.id); for (const { body, want } of posts) { - const before = (await io.linesOf(help.id)).length; - await io.post(help.id, body); + const before = (await io.linesOf(conversation)).length; + await io.post(conversation, body); await quiet(app, 120_000); - const after = (await io.linesOf(help.id)).slice(before + 1); + const after = (await io.linesOf(conversation)).slice(before + 1); const who: string[] = []; - for (const seat of members) { - const turns = (await io.conversationsOf(seat, help.id)).flat().filter((m) => m.role === "user"); - if (turns.some((m) => m.text.includes(`: ${body}`))) who.push(seat); + for (const seat of delegates) { + const turns = (await io.conversationsOf(seat, conversation)).flat().filter((m) => m.role === "user"); + if (turns.some((m) => m.text.endsWith(`: ${body}`))) who.push(seat); } const answers = after.filter((l) => l.author !== undefined); - const line = `"${body}" → [${who.join(", ")}] · ${answers.length} line(s): ${answers.map((a) => `${a.author}: ${a.body}`).join(" | ")}`; + const line = `"${body}" → [${who.join(", ")}] · ${answers.length} line(s): ${answers.map((a) => `${a.author}: ${a.text}`).join(" | ")}`; evidence.push(line); console.log(` live ${line}`); - if (who.length !== 1) fail("live", `"${body}" was heard by [${who.join(", ")}] (want one member)`); + if (who.length !== 1) fail("live", `"${body}" was heard by [${who.join(", ")}] (want one delegate)`); else if (want !== undefined && who[0] !== want) fail("live", `"${body}" went to ${who[0]} (want ${want})`); - if (answers.length !== 1 || answers[0]!.author !== who[0]) fail("live", `"${body}" got ${answers.length} answer line(s) (want one, by ${who[0] ?? "its member"})`); + if (answers.length !== 1 || answers[0]!.author !== who[0]) fail("live", `"${body}" got ${answers.length} answer line(s) (want one, by ${who[0] ?? "its delegate"})`); } } finally { await app.state.dispose(); @@ -414,13 +373,13 @@ await runGoal(async () => { const evidence: string[] = []; const fail = (leg: string, line: string) => failures.push(`[${leg}] ${line}`); - // ---- import: the route is there to call ---------------------------------- + // ---- import: the flow is there to register ------------------------------- const workforce = (await import("@flow-state-dev/workforce")) as Record; - if (typeof workforce.routeByPurpose !== "function") { - fail("import", "@flow-state-dev/workforce exports no routeByPurpose"); + if (typeof workforce.defineCoordinatorFlow !== "function") { + fail("import", "@flow-state-dev/workforce exports no defineCoordinatorFlow"); return { failures, evidence: "" }; } - evidence.push("routeByPurpose resolves from @flow-state-dev/workforce"); + evidence.push("defineCoordinatorFlow resolves from @flow-state-dev/workforce"); // ---- source: what the host writes ---------------------------------------- const builds = /\b(dispatcher|keyedRouter|router)\s*\(/; @@ -429,11 +388,11 @@ await runGoal(async () => { const foreign = imports.filter((spec) => !spec.startsWith("@flow-state-dev/")); if (foreign.length > 0) fail("source", `host.mts imports from outside @flow-state-dev/*: ${foreign.join(", ")}`); if (builds.test(host)) fail("source", `host.mts builds its own ${builds.exec(host)![1]}`); - if (!/routeByPurpose\s*\(/.test(host)) fail("source", "host.mts never calls routeByPurpose"); - const routed = (await readMailboxesDirectory(TREE)).mailboxes.filter((c) => c.declared.routing !== undefined); - if (routed.length !== 1) fail("source", `the tree declares routing: on ${routed.length} mailboxes (want 1)`); + if (!/defineCoordinatorFlow\s*\(/.test(host)) fail("source", "host.mts never registers defineCoordinatorFlow"); + const routed = (await readWorkforce(TREE)).workers.filter((w) => w.declared.flow === "coordinator" && w.declared.routing === "best-fit"); + if (routed.length !== 1) fail("source", `the tree declares routing: best-fit on ${routed.length} coordinators (want 1)`); if (!failures.some((f) => f.startsWith("[source]"))) { - evidence.push(`host.mts imports only ${[...new Set(imports)].join(", ")}, calls routeByPurpose, and builds no dispatcher or router`); + evidence.push(`host.mts imports only ${[...new Set(imports)].join(", ")}, registers defineCoordinatorFlow, and builds no dispatcher or router`); } await scriptedLegs(fail, evidence); From 8a2d8d3f6eb9c224c557f0d274dc763cf6041aa3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 01:23:09 +0000 Subject: [PATCH 02/14] feat(goals): pentest-lab and the kitchen-sink talk goals run on coordinators (FIX-1792 P1) S4 and the rest of S6's goals. - pentest-lab: `pentest.findings` and the four refusal trees' `findings` are coordinator WORKER.md files on `routing: everyone`; the unknown-member scenario's `briefing` names `pentest.nobody` and is now refused at load (BR-10). The host registers Workforce's coordinator flow over the lab's `probe` kind, which takes delegated posts through `delegatedPostEntry`; notify.mts goes. The gate is rewritten on the conversation, the routing record and the delegate sessions, with its twelve controls re-expressed; the honesty check converts. - kitchen-sink-talk: a-post-runs, agent-replies, keeps-both-sides and shows-the-reply convert; answers-a-clerk-note and a-person-talks are rewritten without the escalation legs; lists-a-filed-case retires (goals/README.md). goals/lib/kitchen-sink.mts gains the helpers they share (Talk, the worker copy, the coordinator's conversation). - workforce-mailboxes: fresh-host and routed-post verdict rows. - The inventory checker marks P1's files done and classifies what main added since the spec. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- goals/README.md | 4 + .../fixtures/input.json | 14 +- .../fixtures/today-main.json | 36 - .../run.mts | 410 ++++------ .../fixtures/input.json | 2 +- .../goal.md | 33 +- .../run.mts | 129 ++- .../fixtures/input.json | 2 +- .../agent-replies-in-the-mailbox/goal.md | 30 +- .../agent-replies-in-the-mailbox/run.mts | 213 +++-- .../fixtures/input.json | 11 +- .../answers-a-clerk-note-or-files-it/goal.md | 39 +- .../answers-a-clerk-note-or-files-it/run.mts | 321 ++++---- .../fixtures/input.json | 2 +- .../keeps-both-sides-across-a-reload/goal.md | 30 +- .../keeps-both-sides-across-a-reload/run.mts | 138 ++-- .../fixtures/input.json | 11 - .../goal.md | 42 - .../run.mts | 398 ---------- .../fixtures/input.json | 2 +- .../shows-the-reply-without-a-reload/goal.md | 20 +- .../shows-the-reply-without-a-reload/run.mts | 106 +-- goals/lib/kitchen-sink.mts | 86 +- .../fixtures/input.json | 16 +- .../goal.md | 73 +- .../run.mts | 735 ++++++++---------- .../fixtures/input.json | 2 +- .../goal.md | 14 +- .../harness.mts | 41 +- .../run.mts | 69 +- goals/pentest-lab/lab/README.md | 53 +- goals/pentest-lab/lab/fsdev.config.mts | 6 +- goals/pentest-lab/lab/host.mts | 579 +++++++------- goals/pentest-lab/lab/notify.mts | 184 ----- .../MAILBOX.md => workers/findings/WORKER.md} | 4 +- .../MAILBOX.md => workers/findings/WORKER.md} | 4 +- .../MAILBOX.md => workers/findings/WORKER.md} | 4 +- .../MAILBOX.md => workers/findings/WORKER.md} | 4 +- .../pentest/mailboxes/briefing/MAILBOX.md | 7 - .../teams/pentest/workers/briefing/WORKER.md | 10 + .../lab/workforce/flows/workers/probe.mts | 195 ++--- .../teams/audit/workers/scribe/WORKER.md | 4 +- .../MAILBOX.md => workers/findings/WORKER.md} | 4 +- .../goal.md | 6 + .../a-routed-post-gets-one-answer/goal.md | 8 + specs/issues/FIX-1792/poc/inventory/check.mjs | 51 +- 46 files changed, 1736 insertions(+), 2416 deletions(-) delete mode 100644 goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/fixtures/today-main.json delete mode 100644 goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/fixtures/input.json delete mode 100644 goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/goal.md delete mode 100644 goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/run.mts delete mode 100644 goals/pentest-lab/lab/notify.mts rename goals/pentest-lab/lab/refusal-trees/unknown-kind-corrected/teams/pentest/{mailboxes/findings/MAILBOX.md => workers/findings/WORKER.md} (51%) rename goals/pentest-lab/lab/refusal-trees/unknown-kind/teams/pentest/{mailboxes/findings/MAILBOX.md => workers/findings/WORKER.md} (51%) rename goals/pentest-lab/lab/refusal-trees/unknown-tool-corrected/teams/pentest/{mailboxes/findings/MAILBOX.md => workers/findings/WORKER.md} (51%) rename goals/pentest-lab/lab/refusal-trees/unknown-tool/teams/pentest/{mailboxes/findings/MAILBOX.md => workers/findings/WORKER.md} (51%) delete mode 100644 goals/pentest-lab/lab/scenario-trees/unknown-member/teams/pentest/mailboxes/briefing/MAILBOX.md create mode 100644 goals/pentest-lab/lab/scenario-trees/unknown-member/teams/pentest/workers/briefing/WORKER.md rename goals/pentest-lab/lab/workforce/teams/pentest/{mailboxes/findings/MAILBOX.md => workers/findings/WORKER.md} (69%) diff --git a/goals/README.md b/goals/README.md index 8e7d3ee39e..b93004ac46 100644 --- a/goals/README.md +++ b/goals/README.md @@ -168,3 +168,7 @@ Retired by FIX-1788, when workers became data on one copy per worker flow. Hirin - `mailbox-boards/it-hands-a-task-to-a-fresh-hire`: a task on a board names a standard worker, and its session is opened naming that worker (BR-18). A user's own worker isn't handed board tasks. - `workforce-seats/two-seats-run-their-own-configuration`: a worker is no copy addressable by its own id; each flow is registered once and a session names its worker (BR-26). That each worker's turn runs on its own record's settings and none of its sibling's is proven by `workforce-seats/a-callers-own-agent-wins-every-seat` (b) and `workforce-seats/a-non-agent-seat-receives-its-skills` (b) and (c), and the admission control pair by the latter's (d) and (e). - `workforce-conventions/durable-hire-survives-redeploy`: a hire is a row every turn reads, on any process, so there is nothing to restore after a redeploy (BR-1, BR-20), proven by `workers-as-resources/keeps-each-users-workers-their-own` leg a's second host. + +Retired by FIX-1792, when a mailbox became a coordinator worker file: + +- `kitchen-sink-talk/lists-a-filed-case-without-a-reload`: the escalation feature is removed, not replaced (FIX-1792 BR-14; product owner, 2026-10-06), so nothing files a case and there is no list to show one (BR-24). It last passed on `main` at 57d2c10e9. diff --git a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/fixtures/input.json b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/fixtures/input.json index 1f7385d64d..c04af1f0b8 100644 --- a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/fixtures/input.json +++ b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/fixtures/input.json @@ -1,7 +1,7 @@ { "port": 3043, "person": "devuser", - "mailbox": { "kind": "mailbox", "id": "support.help", "board": "escalations" }, + "coordinator": { "kind": "coordinator", "id": "support.help" }, "seats": { "devices": { "kind": "agent", "id": "support.devices" }, "accounts": { "kind": "agent", "id": "support.accounts" }, @@ -9,7 +9,6 @@ "general": { "kind": "agent", "id": "support.general" } }, "fallback": "general", - "escalatesTo": "accounts", "markers": { "route": "[route:{seat}]", "textAnswer": "[scenario:wake-after-a-hold]", @@ -19,9 +18,7 @@ "talk": "[scenario:talk-to-seat]", "talkReply": "[reply:talk-to-seat]", "recall": "[scenario:recall]", - "recallReply": "[reply:recall]", - "needsAPerson": "[scenario:needs-a-person]", - "filedReply": "[reply:escalated]" + "recallReply": "[reply:recall]" }, "lineWithinMs": 15000, "providerKeys": [ @@ -36,12 +33,7 @@ "withinMs": 120000, "posts": [ { "text": "Hi, my laptop won't join the office wifi since this morning.", "want": "devices" }, - { - "text": "It sees it. It fails right after the password.", - "want": "devices", - "followUpOf": 0, - "subject": ["wifi", "wi-fi", "network", "password", "laptop", "ssid"] - }, + { "text": "It sees it. It fails right after the password." }, { "text": "My phone stopped charging, and it's a brand new cable.", "want": "devices" }, { "text": "Different thing: I was charged twice for my subscription this month.", "want": "accounts" }, { "text": "How do I make a generator return structured output in flow-state-dev?", "want": "fsd" }, diff --git a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/fixtures/today-main.json b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/fixtures/today-main.json deleted file mode 100644 index dc650a29f4..0000000000 --- a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/fixtures/today-main.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "port": 3043, - "person": "devuser", - "mailbox": { "kind": "mailbox", "id": "support.desk", "board": "escalations" }, - "seats": { - "devices": { "kind": "agent", "id": "support.otto" }, - "accounts": { "kind": "agent", "id": "support.iris" }, - "fsd": { "kind": "desk-clerk", "id": "support.grace" }, - "general": { "kind": "followup-runner", "id": "support.wren" } - }, - "fallback": "general", - "escalatesTo": "accounts", - "markers": { - "route": "[route:{seat}]", - "textAnswer": "[scenario:wake]", - "textReply": "[reply:wake]", - "toolAnswer": "[scenario:reply-in-mailbox]", - "toolReply": "[reply:in-mailbox]", - "talk": "[scenario:talk-to-seat]", - "talkReply": "[reply:talk-to-seat]", - "recall": "[scenario:recall]", - "recallReply": "[reply:recall]", - "needsAPerson": "[scenario:needs-a-person]", - "filedReply": "[reply:escalated]" - }, - "lineWithinMs": 15000, - "providerKeys": [ - "AI_GATEWAY_API_KEY", - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", - "OPENROUTER_API_KEY", - "GOOGLE_GENERATIVE_AI_API_KEY", - "XAI_API_KEY" - ], - "smoke": { "withinMs": 120000, "posts": [] } -} diff --git a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/run.mts b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/run.mts index b3a2310380..b94428a543 100644 --- a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/run.mts +++ b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/run.mts @@ -1,9 +1,9 @@ /** - * Closure goal check for the epic FIX-1592 (FIX-1601): a person asks the - * support mailbox a question and, without reloading, sees the one specialist - * whose purpose fits start work and then answer; each specialist keeps only - * its own cases; a direct conversation remembers its own turns and stays out - * of the mailbox; a case that needs a person is filed. + * Closure goal check for the epic FIX-1592 (FIX-1601), on the coordinator + * (FIX-1792): a person asks the support desk's coordinator a question and, + * without reloading, sees the one specialist whose purpose fits start work and + * then answer; each specialist keeps only its own cases; a direct conversation + * remembers its own turns and stays out of the coordinator's conversation. * * Real path, scripted model, keyless, out of CI. See goal.md for the contract. * @@ -12,25 +12,26 @@ * end, then reads again. Everything graded is read off the page, by this run's * tokens: * - * a `[route:]`, a text answer that never calls the post tool, - * token A: ` is working`, then one line by it under the post - * within 15 s, and the row clears. Nobody else works or answers. + * a `[route:]`, a text answer, token A: ` is working`, + * then one line by it under the post within 15 s, and the row clears. + * Nobody else works or answers. * b three posts, each after the previous answer: B1 to devices, B2 to - * accounts, B3 unmarked for the fallback, each answered through the post - * tool: the right seat works, then one line each, by it, and its row - * clears before the next post goes; nobody else works. Each seat's - * conversation in the mailbox holds only its own posts. - * c1 "New conversation" on devices, token C1: the person's turn + * accounts, B3 unmarked for the fallback: the right seat works, then one + * line each, by it, and its row clears before the next post goes; + * nobody else works. Each seat's session for the conversation holds + * only its own posts. + * c1 "Talk" on devices in the roster, token C1: the person's turn * (`c1:turn`), a reply under it (`c1:reply`). * c2 there, the recall scenario, token C2: the person's turn (`c2:turn`), * and a reply naming C1 and no token from a or b (`c2:recall`). The - * mailbox shows none of it (`c2:mailbox`). - * seg (part 4) devices' conversation in the mailbox holds nothing of C1. - * e (part 2) a needs-a-person post, token E: the specialist's line says - * it filed (`e:line`); the team panel's escalations list shows one - * row carrying E on the open page within 15 s of that line - * (`e:row-open`) and after the reload (`e:row`); the boot still warns - * that nothing drains escalations (`e:warning`). + * conversation with the coordinator shows none of it + * (`c2:coordinator`). + * seg (part 4) devices' session for the conversation holds nothing of C1. + * + * Which of a specialist's sessions are its runs for the conversation is read + * from the server, by the session's `workerId`, as an index; the page must + * list each as a run of the conversation, and what each holds is read off the + * page. * * With no GOAL_CONTROL the run takes the plain journey, then each control on a * fresh server and a fresh browser context, all on the one build. Each control @@ -39,7 +40,6 @@ * * Run: PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/run.mts * One: GOAL_CONTROL= on the same command runs that control alone - * Main: GOAL_CONTROL=main, from a checkout of today's `main` with this directory copied in * Smoke: GOAL_LIVE=1 with AI_GATEWAY_API_KEY: the real-model smoke, out of test mode */ import { execFileSync } from "node:child_process"; @@ -49,13 +49,16 @@ import { loadFixture, runGoal } from "../../lib/index.mts"; import { buildKitchenSink, conversation, - newConversation, - open, + coordinatorConversation, + openWorkerCopy, panel, rail, readUntil, - row, + showCoordinator, startKitchenSink, + talkTo, + workerSessions, + WORKER_FLOW, type KitchenSinkServer, } from "../../lib/kitchen-sink.mts"; import { launchChromium } from "../../lib/playwright.mts"; @@ -69,24 +72,18 @@ interface Seat { interface SmokePost { text: string; - /** The specialist whose purpose the post names. */ + /** The specialist whose purpose the post names; none for a post any specialist may take. */ want?: Role; - /** For a follow-up: the index of the post it leans on. */ - followUpOf?: number; - /** Words any one of which the follow-up's answer names its subject by. */ - subject?: string[]; } interface Fixture { port: number; /** How the page labels the person's own lines. */ person: string; - mailbox: Seat & { board: string }; + coordinator: Seat; seats: Record; - /** The mailbox's `routing: fallback:`, as a role. */ + /** The coordinator's `fallback:`, as a role. */ fallback: Role; - /** Who part 2's case is routed to. */ - escalatesTo: Role; markers: { /** `{seat}` is replaced by the seat's id. */ route: string; @@ -98,8 +95,6 @@ interface Fixture { talkReply: string; recall: string; recallReply: string; - needsAPerson: string; - filedReply: string; }; lineWithinMs: number; providerKeys: string[]; @@ -108,48 +103,45 @@ interface Fixture { const CONTROL = process.env.GOAL_CONTROL ?? ""; const LIVE = process.env.GOAL_LIVE === "1"; -// Names are inputs: today's `main` runs on its own roster. -const fixture = loadFixture(import.meta.url, CONTROL === "main" ? "today-main.json" : "input.json"); +const fixture = loadFixture(import.meta.url); const M = fixture.markers; const SEATS = fixture.seats; const ROLES = Object.keys(SEATS) as Role[]; -const MAILBOX = fixture.mailbox.id; +const COORDINATOR = fixture.coordinator.id; /** A post's route marker for one specialist. */ const route = (role: Role) => M.route.replace("{seat}", SEATS[role].id); /** - * The controls, as PLAN → Controls has them. `fail` names the assertions that - * are each control's own signal, every one of which it must redden; `green` - * names the legs it must leave wholly green. Any other red assertion in a leg - * `fail` touches is reported, not counted, and so is a leg in neither list. - * `how` is where the control acts: the server (`GOAL_CONTROL` on `next start`, - * honoured only in test mode), the page (`?goalControl=`), the page's network - * (this run), or a separate checkout. + * The controls. `fail` names the assertions that are each control's own + * signal, every one of which it must redden; `green` names the legs it must + * leave wholly green. Any other red assertion in a leg `fail` touches is + * reported, not counted, and so is a leg in neither list. `how` is where the + * control acts: the server (`GOAL_CONTROL` on `next start`, honoured only in + * test mode), the page (`?goalControl=`), or the page's network (this run). */ -const CONTROLS: Record = { - // The live view and the route: nobody shows as working, a's answer never shows unreloaded, B2 and B3 reach the wrong seat. - main: { how: "checkout", fail: ["a:working", "a:line", "b:line"], green: ["c1"] }, - // No live view: no answer and no filed row until a reload. Not the working rows: the page still re-reads its - // runs when the person posts, so whether a row shows (and then never clears) depends on which read wins. - "no-live": { how: "page", fail: ["a:line", "b:line", "e:row-open"], green: ["c1", "c2", "seg"] }, - // A text answer never lands without a post-tool call. - "no-landing": { how: "server", fail: ["a:line", "e:line"], green: ["b", "c1", "c2", "seg"] }, - // Every member wakes: others work, and the one-line answers and the one filed row multiply or never land. - "no-route": { how: "server", fail: ["a:line", "a:alone", "b:line", "b:alone", "e:line", "e:row-open"], green: ["c1", "c2", "seg"] }, - // The page never gets the person's turns. - "drop-user-message": { how: "network", fail: ["c1:turn", "c2:turn"], green: ["a", "b", "seg", "e"] }, +const CONTROLS: Record = { + // No live view: no answer until a reload. Not the working rows: the page still re-reads its runs when the + // person posts, so whether a row shows (and then never clears) depends on which read wins. + "no-live": { how: "page", fail: ["a:line", "b:line"], green: ["c1", "c2", "seg"] }, + // The specialist answers in its own session and the answer never comes back to the conversation. + "no-landing": { how: "server", fail: ["a:line", "b:line"], green: ["c1", "c2", "seg"] }, + // Best fit read as `everyone`: every delegate works on every post and each lands a line. + "no-route": { how: "server", fail: ["a:line", "a:alone", "b:line", "b:alone"], green: ["c1", "c2", "seg"] }, + // The page never gets the person's turns in the direct conversation. + "drop-user-message": { how: "network", fail: ["c1:turn", "c2:turn"], green: ["a", "b", "seg"] }, // The seat is sent no earlier turn, so the recall cannot name C1. - "no-history": { how: "server", fail: ["c2:recall"], green: ["a", "b", "c1", "seg", "e"] }, - // Nothing is filed, so no row, open or after the reload. - "no-filing": { how: "server", fail: ["e:row-open", "e:row"], green: ["a", "b", "c1", "c2", "seg"] }, + "no-history": { how: "server", fail: ["c2:recall"], green: ["a", "b", "c1", "seg"] }, }; /** Every leg a control's lists can name. */ -const LEGS = ["a", "b", "c1", "c2", "seg", "e"]; +const LEGS = ["a", "b", "c1", "c2", "seg"]; if (CONTROL !== "" && CONTROLS[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(CONTROLS).join(", ")}`); } +/** The origin of the journey in progress, for the server's index. */ +let ORIGIN = ""; + const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const secs = (from: number, to: number | undefined) => (to === undefined ? "never" : `+${((to - from) / 1000).toFixed(1)}s`); @@ -166,7 +158,8 @@ const DROP_KEY = "goal-drop-user-message"; * the action streams that talk to it, and the live session stream, which * Playwright's routing cannot filter because it never ends. What the page sees * when the server keeps no person turn. Scoped to leg c's conversation: a - * seat hears a mailbox post as a `user` turn too. + * seat hears a delegated post as a `user` turn too, and the person's posts to + * the coordinator are `user` messages. * * Plain JavaScript in a string: a function handed to Playwright is compiled * by tsx first, which adds helpers the page does not have. @@ -242,12 +235,8 @@ async function reload(page: Page): Promise { await ready(page); } -/** Show the mailbox's panel, from the rail, without leaving the page. */ -async function showMailbox(page: Page): Promise { - await open(page, fixture.mailbox.kind); - await row(page, MAILBOX).click(); - await panel(page).getByTestId("mailbox-transcript").waitFor({ timeout: 15_000 }); -} +/** Show the person's conversation with the coordinator, from the rail, without leaving the page. */ +const showHelpDesk = (page: Page) => showCoordinator(page, COORDINATOR); interface Line { label: string; @@ -261,15 +250,15 @@ interface Reading { lines: Line[]; } -/** The mailbox's panel as drawn: its working rows and every line. */ -async function readMailbox(page: Page): Promise { +/** The conversation's panel as drawn: its working rows and every line. */ +async function readHelpDesk(page: Page): Promise { const drawn = panel(page); const [working, lines] = await Promise.all([ drawn.getByTestId("working-row").allTextContents(), - drawn.getByTestId("mailbox-line").evaluateAll((els) => + drawn.getByTestId("coordinator-line").evaluateAll((els) => els.map((el) => ({ - label: el.querySelector('[data-testid="mailbox-line-label"]')?.textContent ?? "", - body: el.querySelector('[data-testid="mailbox-line-body"]')?.textContent ?? "", + label: el.querySelector('[data-testid="coordinator-line-label"]')?.textContent ?? "", + body: el.querySelector('[data-testid="coordinator-line-body"]')?.textContent ?? "", })), ), ]); @@ -296,9 +285,9 @@ function answersTo(lines: Line[], token: string): Line[] | undefined { return next === -1 ? rest : rest.slice(0, next); } -/** Post a line from the mailbox's panel once its composer is free. Returns when Send was pressed. */ +/** Post a line from the conversation's panel once its composer is free. Returns when Send was pressed. */ async function postLine(page: Page, text: string): Promise { - const box = panel(page).getByLabel("Post to this mailbox"); + const box = panel(page).getByLabel("Post to this coordinator"); await readUntil(async () => (await box.isEnabled()) && (await box.inputValue()) === "", (free) => free, 30_000); await box.fill(text); await panel(page).getByRole("button", { name: "Send" }).click(); @@ -314,7 +303,7 @@ interface Followed { } /** - * Read the mailbox's panel from Send until `answered`, or the time is up; + * Read the conversation's panel from Send until `answered`, or the time is up; * then until no working row is left, for at most `clearMs`. Every reading is * kept, so who showed as working, and when, is graded off them. */ @@ -322,7 +311,7 @@ async function follow(page: Page, sentAt: number, answered: (r: Reading) => bool const readings: Reading[] = []; let lineAt: number | undefined; while (Date.now() < sentAt + withinMs) { - const r = await readMailbox(page); + const r = await readHelpDesk(page); readings.push(r); if (answered(r)) { lineAt = r.at; @@ -333,7 +322,7 @@ async function follow(page: Page, sentAt: number, answered: (r: Reading) => bool let clearedAt: number | undefined; if (lineAt !== undefined) { for (const until = Date.now() + clearMs; Date.now() < until; await sleep(100)) { - const r = await readMailbox(page); + const r = await readHelpDesk(page); readings.push(r); if (r.working.length === 0) { clearedAt = r.at; @@ -355,48 +344,62 @@ function othersWorking(readings: Reading[], seat: string): string[] { type Message = { role: string; text: string }; -/** The seat's leaf in the rail, re-read now: closed if open, then opened. */ -async function freshLeaf(page: Page, seat: Seat) { - await open(page, seat.kind); - const seatRow = row(page, seat.id); - await seatRow.waitFor({ timeout: 15_000 }); - if ((await seatRow.getAttribute("aria-expanded")) === "true") await seatRow.click(); - await open(page, seat.id); - const leaf = rail(page).locator(`ul[data-leaf="${seat.id}"]`); - await leaf.waitFor({ timeout: 15_000 }); - await readUntil( - async () => (await leaf.locator("[data-session-id]").count()) + (await leaf.getByText("No sessions yet").count()), - (n) => n > 0, - 10_000, - ); - return leaf; +/** The worker copy's leaf in the rail, re-read now: closed if open, then opened. */ +async function freshCopy(page: Page) { + // The flow's kind and its one copy share the name `agent`, so each is found by its own attribute. + const kind = rail(page).locator(`button[data-kind="${WORKER_FLOW}"]`); + await kind.waitFor({ timeout: 15_000 }); + if ((await kind.getAttribute("aria-expanded")) !== "true") await kind.click(); + const copy = rail(page).locator(`button[data-instance-id="${WORKER_FLOW}"]`); + await copy.waitFor({ timeout: 15_000 }); + if ((await copy.getAttribute("aria-expanded")) === "true") await copy.click(); + return await openWorkerCopy(page); } -let lastDrawn = ""; +/** A session's kept message texts, in order, as the server holds them. An index for waiting, never graded. */ +async function keptTexts(page: Page, sessionId: string): Promise { + const res = await page.request.get(`${ORIGIN}/api/flows/sessions/${sessionId}/state?include_items=true&item_types=message&limit=1000`); + const items = ((await res.json()) as { items?: Array<{ content?: Array<{ text?: string }> }> }).items ?? []; + return items.map((item) => (item.content ?? []).map((part) => part.text ?? "").join("").trim()); +} -/** Open one of a seat's listed conversations and read it as drawn, once the panel holds it. */ -async function readConversation(page: Page, seat: Seat, sessionId: string): Promise { - const button = rail(page).locator(`ul[data-leaf="${seat.id}"] [data-session-id="${sessionId}"]`); +/** + * Open one of the listed sessions and read it as drawn, once the panel holds it. + * + * The panel keeps the last session's items until this one's load, and a seat + * panel draws a coordinator conversation's messages as turns too, so "the + * panel changed" is not enough. It waits until what is drawn is this + * session's, by the server's own read of it (not graded); a page that never + * draws it is graded on whatever it drew last. + */ +async function readConversation(page: Page, sessionId: string, runOf?: string): Promise { + const button = rail(page).locator( + `ul[data-leaf="${WORKER_FLOW}"] [data-session-id="${sessionId}"]${runOf === undefined ? "" : `[data-dispatch-run-of="${runOf}"]`}`, + ); + await button.waitFor({ timeout: 10_000 }); await button.click(); await readUntil(() => button.getAttribute("aria-current"), (v) => v === "true", 5_000); - // The panel keeps the last conversation until this one replaces it. - const messages = await readUntil( + const kept = await keptTexts(page, sessionId); + return await readUntil( () => conversation(page), - (ms) => ms.length > 0 && JSON.stringify(ms) !== lastDrawn, + (ms) => ms.length === kept.length && kept.every((text, i) => ms[i]!.text.includes(text)), 8_000, ); - lastDrawn = JSON.stringify(messages); - return messages; } -/** A seat's conversations in the mailbox (its dispatch runs of it), each read as drawn. */ +/** + * A seat's sessions for the person's conversation with the coordinator, each + * read as drawn. Which they are is the server's index by worker; each must be + * listed in the rail as a run of the conversation. + */ async function runsOf(page: Page, seat: Seat): Promise> { - const leaf = await freshLeaf(page, seat); - const ids = await leaf - .locator(`[data-dispatch-run-of="${MAILBOX}"]`) - .evaluateAll((buttons) => buttons.map((b) => b.getAttribute("data-session-id") ?? "")); + const parent = await coordinatorConversation(page, ORIGIN, COORDINATOR); + const ids = (await workerSessions(page, ORIGIN, seat.id, { runs: true })) + .filter((session) => parent !== undefined && session.parentSessionId === parent) + .map((session) => session.id); + await freshCopy(page); const out: Array<{ sessionId: string; messages: Message[] }> = []; - for (const id of ids) out.push({ sessionId: id, messages: await readConversation(page, seat, id) }); + for (const id of ids) out.push({ sessionId: id, messages: await readConversation(page, id, parent) }); return out; } @@ -409,13 +412,6 @@ function replyTo(messages: Message[], token: string): Message | undefined { return (next === -1 ? rest : rest.slice(0, next)).find((m) => m.role === "assistant"); } -/** The rows the team panel draws on the mailbox's board: each row's text. */ -const boardRows = (page: Page) => - page - .getByTestId(`board-${MAILBOX}.${fixture.mailbox.board}`) - .locator("li[data-task-id]") - .evaluateAll((rows) => rows.map((r) => r.textContent ?? "")); - // --------------------------------------------------------------------------- // The legs // --------------------------------------------------------------------------- @@ -430,13 +426,12 @@ interface Tokens { b: [string, string, string]; c1: string; c2: string; - e: string; } -/** Leg a, ask `support`: the routed specialist works, then answers under its name, and nobody else. */ +/** Leg a, ask the coordinator: the routed specialist works, then answers under its name, and nobody else. */ async function legA(page: Page, t: Tokens, fail: Fail, evidence: string[]): Promise { const seat = SEATS.devices.id; - await showMailbox(page); + await showHelpDesk(page); const sentAt = await postLine(page, `${route("devices")} ${M.textAnswer} ${t.a} is my laptop covered for a cracked screen?`); const f = await follow(page, sentAt, (r) => (answersTo(r.lines, t.a)?.length ?? 0) > 0, fixture.lineWithinMs); const workingAt = f.readings.find( @@ -463,9 +458,9 @@ async function legA(page: Page, t: Tokens, fail: Fail, evidence: string[]): Prom } await reload(page); - await showMailbox(page); + await showHelpDesk(page); const after = await readUntil( - () => readMailbox(page), + () => readHelpDesk(page), (r) => postsOf(r.lines, t.a).length > 0 && (answersTo(r.lines, t.a)?.length ?? 0) > 0, 10_000, ); @@ -484,14 +479,14 @@ async function legB(page: Page, t: Tokens, fail: Fail, evidence: string[]): Prom { token: t.b[1], role: "accounts", marked: true, ask: "why was I billed twice?" }, { token: t.b[2], role: fixture.fallback, marked: false, ask: "where do I leave feedback about the office?" }, ]; - await showMailbox(page); + await showHelpDesk(page); let previous = `A (${t.a})`; for (const post of plan) { const seat = SEATS[post.role].id; const name = `${post.token} (${post.marked ? route(post.role) : "unmarked, the fallback"})`; // The last post's row is gone before this one goes. One that stays is that post's, and it is // kept in this post's readings, so an overlap still shows. - const idle = await readUntil(() => readMailbox(page), (r) => r.working.length === 0, 5_000); + const idle = await readUntil(() => readHelpDesk(page), (r) => r.working.length === 0, 5_000); if (idle.working.length > 0) { fail("b:clears", `${previous}: ${JSON.stringify(idle.working)} still showed when ${post.token} was to be sent`); } @@ -532,9 +527,9 @@ async function legB(page: Page, t: Tokens, fail: Fail, evidence: string[]): Prom await gradeConversations(page, "open page", expected, every, fail, evidence); await reload(page); - await showMailbox(page); + await showHelpDesk(page); const after = await readUntil( - () => readMailbox(page), + () => readHelpDesk(page), (r) => plan.every((p) => repliesOf(r.lines, p.token).length > 0), 10_000, ); @@ -548,7 +543,7 @@ async function legB(page: Page, t: Tokens, fail: Fail, evidence: string[]): Prom await gradeConversations(page, "after the reload", expected, every, fail, evidence); } -/** Each seat's conversation in the mailbox holds its own posts once, each answered, and nobody else's. */ +/** Each seat's session for the conversation holds its own posts once, each answered, and nobody else's. */ async function gradeConversations( page: Page, when: string, @@ -570,44 +565,40 @@ async function gradeConversations( const foreign = every.filter((token) => !mine.includes(token)); const text = runs.flatMap((r) => r.messages).map((m) => m.text).join("\n"); const leaked = foreign.filter((token) => text.includes(token)); - if (leaked.length > 0) fail("b:only-its-own", `${when}: ${seat.id}'s conversation in ${MAILBOX} holds ${leaked.join(", ")}, posts it was not routed`); + if (leaked.length > 0) fail("b:only-its-own", `${when}: ${seat.id}'s session for the conversation with ${COORDINATOR} holds ${leaked.join(", ")}, posts it was not routed`); if (mine.length === 0) { - held.push(`${seat.id} none (${runs.length} conversations)`); + held.push(`${seat.id} none (${runs.length} sessions)`); continue; } if (runs.length !== 1) { - fail("b:only-its-own", `${when}: ${seat.id} lists ${runs.length} conversations in ${MAILBOX} (want 1)`); + fail("b:only-its-own", `${when}: ${seat.id} has ${runs.length} sessions for the conversation with ${COORDINATOR} (want 1)`); continue; } const messages = runs[0]!.messages; for (const token of mine) { const heard = messages.filter((m) => m.role === "user" && m.text.includes(token)).length; - if (heard !== 1) fail("b:only-its-own", `${when}: ${seat.id}'s conversation in ${MAILBOX} holds ${token} as a heard turn ${heard} times (want once)`); + if (heard !== 1) fail("b:only-its-own", `${when}: ${seat.id}'s session for the conversation holds ${token} as a heard turn ${heard} times (want once)`); else if (replyTo(messages, token) === undefined) fail("b:only-its-own", `${when}: ${seat.id} heard ${token} and no reply sits under it`); } held.push(`${seat.id} ${mine.join(" ")}`); } - if (red === 0) evidence.push(`b: ${when}, each conversation in ${MAILBOX} holds: ${held.join("; ")}`); + if (red === 0) evidence.push(`b: ${when}, each specialist's session for the conversation with ${COORDINATOR} holds: ${held.join("; ")}`); } /** Leg c, direct talk, and the segmentation row of part 4. */ async function legC(page: Page, t: Tokens, dropUser: boolean, fail: Fail, evidence: string[]): Promise { const seat = SEATS.devices; - await freshLeaf(page, seat); - await newConversation(page, seat.id); + // From the coordinator's panel, which has no seat composer, so the composer + // below is the one "Talk" opens and not the last seat session leg b read. + await showHelpDesk(page); + await talkTo(page, seat.id); const box = panel(page).getByLabel("Message this seat"); await box.waitFor({ timeout: 15_000 }); - const current = await readUntil( - () => - rail(page) - .locator(`ul[data-leaf="${seat.id}"] [aria-current="true"]`) - .evaluateAll((els) => els.map((el) => el.getAttribute("data-session-id") ?? "")), - (ids) => ids.length > 0, - 10_000, - ); - const sessionId = current[0]; + // Which session "Talk" opened, from the server's index by worker: the person's own, not a run. Not graded. + const started = await readUntil(() => workerSessions(page, ORIGIN, seat.id), (listed) => listed.length > 0, 10_000); + const sessionId = started.length === 1 ? started[0]!.id : undefined; if (sessionId === undefined) { - fail("c1:setup", `"New conversation" on ${seat.id} opened no conversation in the rail`); + fail("c1:setup", `"Talk" on ${seat.id} left the person ${started.length} conversations with it (want 1)`); return; } if (dropUser) { @@ -652,25 +643,25 @@ async function legC(page: Page, t: Tokens, dropUser: boolean, fail: Fail, eviden // ---- c2: recall, answered from the earlier turn and nothing else --------- const two = await talk(`${M.recall} ${t.c2} what did I tell you before this?`, t.c2, M.recallReply); gradeC2(two.messages, two.most, t, "open page", fail, evidence, `${secs(two.sentAt, two.heardAt)}${two.note}`); - await showMailbox(page); - gradeMailboxQuiet((await loaded(page, t)).lines, t, "open page", fail, evidence); + await showHelpDesk(page); + gradeDeskQuiet((await loaded(page, t)).lines, t, "open page", fail, evidence); // ---- the reload at the leg's end ---------------------------------------- await reload(page); - await freshLeaf(page, seat); - const kept = await readConversation(page, seat, sessionId); + await freshCopy(page); + const kept = await readConversation(page, sessionId); gradeC1(kept, kept.filter((m) => m.role === "user" && m.text.includes(t.c1)).length, t, "after the reload", fail, evidence, ""); gradeC2(kept, kept.filter((m) => m.role === "user" && m.text.includes(t.c2)).length, t, "after the reload", fail, evidence, ""); - await showMailbox(page); - gradeMailboxQuiet((await loaded(page, t)).lines, t, "after the reload", fail, evidence); + await showHelpDesk(page); + gradeDeskQuiet((await loaded(page, t)).lines, t, "after the reload", fail, evidence); - // ---- seg (part 4): the mailbox conversation holds nothing of the direct talk + // ---- seg (part 4): the specialist's session for the conversation holds nothing of the direct talk const runs = await runsOf(page, seat); const text = runs.flatMap((r) => r.messages).map((m) => m.text).join("\n"); if (text.includes(t.c1) || text.includes(t.c2)) { - fail("seg", `${seat.id}'s conversation in ${MAILBOX} holds the direct talk (${[t.c1, t.c2].filter((x) => text.includes(x)).join(", ")})`); + fail("seg", `${seat.id}'s session for the conversation with ${COORDINATOR} holds the direct talk (${[t.c1, t.c2].filter((x) => text.includes(x)).join(", ")})`); } else { - evidence.push(`seg: ${seat.id}'s conversation in ${MAILBOX} (${runs.length} run) holds nothing of ${t.c1} or ${t.c2}`); + evidence.push(`seg: ${seat.id}'s session for the conversation with ${COORDINATOR} (${runs.length} run) holds nothing of ${t.c1} or ${t.c2}`); } } @@ -697,90 +688,24 @@ function gradeC2(messages: Message[], most: number, t: Tokens, when: string, fai } else if (!reply.text.includes(t.c1)) { fail("c2:recall", `${when}: the recall reply ${JSON.stringify(reply.text)} does not name ${t.c1}, the conversation's earlier turn`); } else if (earlier.length > 0) { - fail("c2:recall", `${when}: the recall reply names ${earlier.join(", ")}, from the mailbox: ${JSON.stringify(reply.text)}`); + fail("c2:recall", `${when}: the recall reply names ${earlier.join(", ")}, from the conversation with ${COORDINATOR}: ${JSON.stringify(reply.text)}`); } else { evidence.push(`c2: ${when}, ${t.c2} is the person's turn${timing === "" ? "" : ` (${timing})`}, ${JSON.stringify(reply.text)} under it`); } } /** - * The mailbox's panel once its lines have loaded: once the person's posts + * The conversation's panel once its lines have loaded: once the person's posts * from legs a and b are drawn. Waiting only; what it shows is graded by the caller. */ const loaded = (page: Page, t: Tokens) => - readUntil(() => readMailbox(page), (r) => [t.a, ...t.b].every((token) => postsOf(r.lines, token).length > 0), 10_000); + readUntil(() => readHelpDesk(page), (r) => [t.a, ...t.b].every((token) => postsOf(r.lines, token).length > 0), 10_000); -/** `support` shows nothing of the direct talk. */ -function gradeMailboxQuiet(lines: Line[], t: Tokens, when: string, fail: Fail, evidence: string[]): void { +/** The conversation with the coordinator shows nothing of the direct talk. */ +function gradeDeskQuiet(lines: Line[], t: Tokens, when: string, fail: Fail, evidence: string[]): void { const leaks = lines.filter((l) => [t.c1, t.c2, M.talkReply, M.recallReply].some((x) => l.body.includes(x))); - if (leaks.length > 0) fail("c2:mailbox", `${when}: ${MAILBOX} shows the direct talk: ${JSON.stringify(leaks.map((l) => `${l.label}: ${l.body}`))}`); - else evidence.push(`c2: ${when}, ${MAILBOX} shows nothing of it (${lines.length} lines)`); -} - -/** Part 2, the escalation: the specialist files the case onto the board and says so. */ -async function partTwo(page: Page, origin: string, t: Tokens, fail: Fail, evidence: string[]): Promise { - const seat = SEATS[fixture.escalatesTo].id; - const board = fixture.mailbox.board; - await showMailbox(page); - const rowsBefore = (await boardRows(page)).length; - const sentAt = await postLine( - page, - `${route(fixture.escalatesTo)} ${M.needsAPerson} ${t.e} I was charged three times this month and need a person to reverse it`, - ); - const f = await follow(page, sentAt, (r) => (answersTo(r.lines, t.e)?.length ?? 0) > 0, fixture.lineWithinMs); - const answers = answersTo(f.readings.at(-1)!.lines, t.e) ?? []; - if (answers.length !== 1 || answers[0]!.label !== seat || !answers[0]!.body.includes(M.filedReply)) { - fail("e:line", `open page: the lines answering ${t.e} are ${JSON.stringify(answers.map((l) => `${l.label}: ${l.body}`))} (want one, by ${seat}, carrying ${M.filedReply}), line ${secs(sentAt, f.lineAt)}`); - } else { - evidence.push(`e: open page, ${seat}'s line ${secs(sentAt, f.lineAt)}: ${JSON.stringify(answers[0]!.body)}`); - } - // The row, on the open page: the list shows it without a reload, within 15 s - // of the line, counted from the reading that first showed the line (or from - // when the line was due, if none did), not from when `follow` stopped. - const rowFrom = f.lineAt ?? sentAt + fixture.lineWithinMs; - let openRows: string[] = []; - let rowAt: number | undefined; - while (Date.now() <= rowFrom + fixture.lineWithinMs) { - const at = Date.now(); - openRows = await boardRows(page); - if (openRows.some((r) => r.includes(t.e))) { - rowAt = at; - break; - } - await sleep(250); - } - const openMine = rowAt === undefined ? [] : openRows.filter((r) => r.includes(t.e)); - if (openMine.length !== 1) { - fail("e:row-open", `open page: the team panel's ${board} list holds ${openMine.length} rows carrying ${t.e} within ${fixture.lineWithinMs / 1000}s of its line, with no reload (want 1); it shows ${openRows.length} rows, ${rowsBefore} when the leg began`); - } else { - evidence.push(`e: open page, the ${board} list shows ${JSON.stringify(openMine[0])} ${secs(rowFrom, rowAt)} after its line`); - } - // The row is written by the mailbox's own request, a moment after the - // dispatch. Let it land before the reload; nothing here is graded. - await readUntil( - async () => (await page.request.get(`${origin}/api/flows/sessions/${MAILBOX}/resources/${MAILBOX}.${board}`)).text(), - (body) => body.includes(t.e), - 10_000, - ); - - await reload(page); - await showMailbox(page); - const after = await readUntil( - () => readMailbox(page), - (r) => (answersTo(r.lines, t.e)?.length ?? 0) > 0, - 10_000, - ); - const kept = answersTo(after.lines, t.e) ?? []; - if (kept.length !== 1 || kept[0]!.label !== seat || !kept[0]!.body.includes(M.filedReply)) { - fail("e:line", `after the reload, the lines answering ${t.e} are ${JSON.stringify(kept.map((l) => `${l.label}: ${l.body}`))} (want one, by ${seat}, carrying ${M.filedReply})`); - } - const rows = await readUntil(() => boardRows(page), (rs) => rs.some((r) => r.includes(t.e)), 10_000); - const mine = rows.filter((r) => r.includes(t.e)); - if (mine.length !== 1) { - fail("e:row", `after the reload, the team panel's ${board} list holds ${mine.length} rows carrying ${t.e} (want 1); it shows ${rows.length} rows`); - } else { - evidence.push(`e: after the reload, the ${board} list shows ${JSON.stringify(mine[0])}`); - } + if (leaks.length > 0) fail("c2:coordinator", `${when}: the conversation with ${COORDINATOR} shows the direct talk: ${JSON.stringify(leaks.map((l) => `${l.label}: ${l.body}`))}`); + else evidence.push(`c2: ${when}, the conversation with ${COORDINATOR} shows nothing of it (${lines.length} lines)`); } // --------------------------------------------------------------------------- @@ -825,7 +750,6 @@ async function journey(browser: Browser, control: string): Promise { b: [`reply-token-b1${run}`, `reply-token-b2${run}`, `reply-token-b3${run}`], c1: `talk-token-c${run}`, c2: `recall-token-c${run}`, - e: `case-token-e${run}`, }; const failures: string[] = []; failuresSoFar = failures; @@ -837,15 +761,14 @@ async function journey(browser: Browser, control: string): Promise { AI_GATEWAY_API_KEY: "", GOAL_CONTROL: spec?.how === "server" ? control : "", }); + ORIGIN = server.origin; const context = await browser.newContext({ viewport: { width: 1440, height: 900 } }); if (spec?.how === "network") await context.addInitScript({ content: DROP_USER_MESSAGES }); const page = await context.newPage(); - lastDrawn = ""; const legs: Array<[string, () => Promise]> = [ ["a", () => legA(page, t, fail, evidence)], ["b", () => legB(page, t, fail, evidence)], ["c1", () => legC(page, t, spec?.how === "network", fail, evidence)], - ["e", () => partTwo(page, server.origin, t, fail, evidence)], ]; try { await page.goto(`${server.origin}/${spec?.how === "page" ? `?goalControl=${control}` : ""}`); @@ -865,13 +788,6 @@ async function journey(browser: Browser, control: string): Promise { await stop(server); } - // Nobody drains escalations: the boot still warns about it, and about no other board. - const unattended = [...new Set([...server.log().matchAll(/mailbox "([^"]+)" holds board "([^"]+)"/g)].map((m) => `${m[1]}.${m[2]}`))]; - if (JSON.stringify(unattended) !== JSON.stringify([`${MAILBOX}.${fixture.mailbox.board}`])) { - fail("e:warning", `the boot warns these boards are unattended: ${JSON.stringify(unattended)} (want only ${MAILBOX}.${fixture.mailbox.board})`); - } else { - evidence.push(`e: the boot still warns that ${MAILBOX}.${fixture.mailbox.board} is unattended, and no other board`); - } // A leg that could not be walked is a finding only with its cause, and the server's output is the one place it shows. if (failures.some((f) => assertionOf(f).endsWith(":setup"))) { const tail = server.log().trimEnd().split("\n").slice(-40).map((line) => ` ${line}`).join("\n"); @@ -880,7 +796,7 @@ async function journey(browser: Browser, control: string): Promise { return { name: control === "" ? "plain" : control, failures, evidence }; } -/** The leg a failure names: `c2` for `c2:recall`, `b`, `e` for `e:row`. */ +/** The leg a failure names: `c2` for `c2:recall`, `b` for `b:line`. */ const legOf = (failure: string) => (/^\[([^\]:]+)/.exec(failure)?.[1] ?? ""); /** Print a journey's verdict per leg, as it ran. */ @@ -890,7 +806,7 @@ function report(j: Journey): void { for (const line of j.failures) console.log(` FAIL ${line}`); } -/** The assertion a failure names: `a:working`, `c2:recall`, `e:row`, `b:setup`. */ +/** The assertion a failure names: `a:working`, `c2:recall`, `b:setup`. */ const assertionOf = (failure: string) => (/^\[([^\]]+)\]/.exec(failure)?.[1] ?? ""); /** @@ -946,6 +862,7 @@ async function smoke(browser: Browser, failures: string[], evidence: string[]): } const run = randomUUID().replace(/-/g, "").slice(0, 10); const server = await startKitchenSink(fixture.port, { KITCHEN_SINK_TEST_MODE: "", GOAL_CONTROL: "" }); + ORIGIN = server.origin; const context = await browser.newContext({ viewport: { width: 1440, height: 900 } }); const page = await context.newPage(); const posts = fixture.smoke.posts.map((p, i) => ({ ...p, mark: `(ref smoke-token-${i}${run})` })); @@ -953,12 +870,12 @@ async function smoke(browser: Browser, failures: string[], evidence: string[]): try { await page.goto(`${server.origin}/`); await ready(page); - await showMailbox(page); + await showHelpDesk(page); const within = `${fixture.smoke.withinMs / 1000}s`; for (const [i, post] of posts.entries()) { const want = post.want === undefined ? undefined : SEATS[post.want].id; // No post goes while a run is working: every answer then belongs to the post it follows. - const idle = await readUntil(() => readMailbox(page), (r) => r.working.length === 0, fixture.smoke.withinMs); + const idle = await readUntil(() => readHelpDesk(page), (r) => r.working.length === 0, fixture.smoke.withinMs); if (idle.working.length > 0) { fail("smoke", `before post ${i + 1}: ${JSON.stringify(idle.working)} still showed after ${within}, so the smoke stops here and post ${i + 1} is not sent`); break; @@ -974,20 +891,11 @@ async function smoke(browser: Browser, failures: string[], evidence: string[]): evidence.push(said); if (f.lineAt === undefined) fail("smoke", `post ${i + 1}: no answer within ${fixture.smoke.withinMs / 1000}s of Send, with no reload`); else if (answers.length !== 1) fail("smoke", `post ${i + 1}: ${answers.length} answer lines (want exactly one)`); + else if (!ROLES.some((role) => SEATS[role].id === answers[0]!.label)) fail("smoke", `post ${i + 1} was answered by ${answers[0]!.label}, not a specialist`); else if (want !== undefined && answers[0]!.label !== want) fail("smoke", `post ${i + 1} ${JSON.stringify(post.text)} was answered by ${answers[0]!.label} (want ${want})`); if (worked.filter((s) => s !== undefined && s !== answers[0]?.label).length > 0) { fail("smoke", `post ${i + 1}: the panel showed ${worked.join(", ")} working (want the answering specialist alone)`); } - if (post.followUpOf !== undefined && answers.length === 1) { - const antecedent = posts[post.followUpOf]!; - const theirs = answeredBy[post.followUpOf]; - const subject = (post.subject ?? []).filter((word) => answers[0]!.body.toLowerCase().includes(word)); - evidence.push(`follow-up: ${JSON.stringify(antecedent.text)} answered by ${theirs?.label ?? "nobody"}: ${JSON.stringify(theirs?.body ?? "")}; ${JSON.stringify(post.text)} answered by ${answers[0]!.label}: ${JSON.stringify(answers[0]!.body)}`); - if (theirs === undefined || answers[0]!.label !== theirs.label) { - fail("smoke", `the follow-up was answered by ${answers[0]!.label}, not ${theirs?.label ?? "its antecedent's specialist"}`); - } - if (subject.length === 0) fail("smoke", `the follow-up's answer names none of ${JSON.stringify(post.subject)}: ${JSON.stringify(answers[0]!.body)}`); - } // A run still working (or no answer to wait out) means the next post could draw this one's answer. if (f.clearedAt === undefined) { const next = i + 1 < posts.length ? `post ${i + 2} is not sent` : "no post follows"; @@ -1003,9 +911,9 @@ async function smoke(browser: Browser, failures: string[], evidence: string[]): // Only the posts that went. const sent = posts.slice(0, answeredBy.length); await reload(page); - await showMailbox(page); + await showHelpDesk(page); const after = await readUntil( - () => readMailbox(page), + () => readHelpDesk(page), (r) => sent.every((p) => postsOf(r.lines, p.mark).length > 0), 15_000, ); @@ -1040,7 +948,7 @@ await runGoal(async () => { await smoke(browser, failures, evidence); return { failures, evidence: evidence.join("; ") }; } - const names = CONTROL !== "" ? [CONTROL] : ["", ...Object.keys(CONTROLS).filter((n) => CONTROLS[n]!.how !== "checkout")]; + const names = CONTROL !== "" ? [CONTROL] : ["", ...Object.keys(CONTROLS)]; let plain: Journey | undefined; for (const name of names) { const j = await journey(browser, name); diff --git a/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/fixtures/input.json b/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/fixtures/input.json index 45e7e49ff1..b9cf2fb061 100644 --- a/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/fixtures/input.json +++ b/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/fixtures/input.json @@ -1,6 +1,6 @@ { "port": 3039, - "mailbox": { "kind": "mailbox", "id": "support.help" }, + "coordinator": { "kind": "coordinator", "id": "support.help" }, "agents": [{ "kind": "agent", "id": "support.accounts" }], "others": [ { "kind": "agent", "id": "support.devices" }, diff --git a/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/goal.md b/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/goal.md index 3f5c0428d0..90f89ada4d 100644 --- a/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/goal.md +++ b/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/goal.md @@ -1,29 +1,31 @@ # kitchen-sink-talk › a post runs each member agent once -**Issue:** FIX-1590 (VG of the spec's PLAN; the epic FIX-1592's leg b); re-pointed by FIX-1611 +**Issue:** FIX-1590 (VG of the spec's PLAN; the epic FIX-1592's leg b); re-pointed by FIX-1611; converted by FIX-1792 P1 -**Outcome:** When a person posts to `support.help` from the kitchen-sink page, the one specialist the mailbox's route picks runs once on the post and no other member runs, and that run is there in the specialist's conversation after a reload. +**Outcome:** When a person posts to `support.help` from the kitchen-sink page, the one specialist the coordinator's best fit picks runs once on the post and no other delegate runs, and that run is there in the specialist's session after a reload. -**Re-pointed (FIX-1611).** The folder name is kept so its verdict history stays in one place. Until FIX-1611 this read `support.desk`, where every agent member ran on each post. The team now has one routed mailbox, so the check asks the question that roster answers: the picked specialist once, nobody else. +**Re-pointed (FIX-1611).** The folder name is kept so its verdict history stays in one place. Until FIX-1611 this read `support.desk`, where every agent member ran on each post. The team now has one routed help desk, so the check asks the question that roster answers: the picked specialist once, nobody else. -**Input:** `fixtures/input.json`: the mailbox, the specialist a post routes to (`support.accounts`), the three members it passes over (`support.devices`, `support.fsd`, `support.general`), the `[route:]` tag the scripted route reads, the scenario marker the scripted model keys on and the reply marker it writes. Held-out: each post carries a fresh token per run, and only that token is graded, so a conversation another run left behind can never be the one that passes. A second post, with different text, must land in the same conversation and pass too. +**Converted (FIX-1792 P1).** `support.help` is a coordinator worker file on `routing: best-fit`, no longer a mailbox. A post is delivered into the picked specialist's own session for the person's conversation (a session on the `agent` flow, naming the specialist, listed in the rail as a run of the conversation). Since FIX-1788 the rail lists every worker's sessions under the `agent` flow's one copy, not a row per specialist, so which sessions are a specialist's is read from the server by the session's `workerId`, as an index; what each holds is still read off the page. The controls follow the coordinator: `name-only-notify` and `no-author-filter` had no coordinator counterpart and go; `no-delivery` and `answers-go-on` take their places. -**Signal:** one real browser against kitchen-sink's **production build** (built by the run, never assumed), served by `next start` on the scripted model, the in-memory store and **no model key**. Everything graded is read off the page **after a reload**, by opening every conversation the seat lists in the rail. +**Input:** `fixtures/input.json`: the coordinator, the specialist a post routes to (`support.accounts`), the three delegates it passes over (`support.devices`, `support.fsd`, `support.general`), the `[route:]` tag the scripted evaluation reads, the scenario marker the scripted model keys on and the reply marker it writes. Held-out: each post carries a fresh token per run, and only that token is graded, so a session another run left behind can never be the one that passes. A second post, with different text, must land in the same session and pass too. -- **support.accounts**: exactly one listed conversation holds the first post's token; it holds the post once, as the seat's `user` turn, with an `assistant` reply carrying `[reply:wake]` right under it. After a second post and a reload, that same conversation holds the second post the same way, and still holds the first. -- **others**: none of `support.devices`', `support.fsd`'s or `support.general`'s conversations holds either token. +**Signal:** one real browser against kitchen-sink's **production build** (built by the run, never assumed), served by `next start` on the scripted model, the in-memory store and **no model key**. Everything graded is read off the page **after a reload**, by opening every session the specialist has from the worker copy in the rail. -**Anti-game:** no assertion on the reply's words past its marker, on the route's output, on the fan-out's output, on a dispatch handle, on the transient name-only line, on a package test or on a CLI run. The seats' conversations are read off the page as drawn, after a reload, so a streamed copy cannot pass. Before each reload the run polls the server until the answer lands; that wait is not graded. +- **support.accounts**: exactly one of its sessions holds the first post's token; it holds the post once, as the specialist's `user` turn, with an `assistant` reply carrying `[reply:wake]` right under it. After a second post and a reload, that same session holds the second post the same way, and still holds the first. +- **others**: none of `support.devices`', `support.fsd`'s or `support.general`'s sessions holds either token. -**Model:** n/a. kitchen-sink's scripted model answers the seats and picks the route (epic FIX-1592 D3). The goal is who ran, not what they said. Keyless: the server runs with `AI_GATEWAY_API_KEY` empty. +**Anti-game:** no assertion on the reply's words past its marker, on the routing record, on a dispatch handle, on a package test or on a CLI run. The specialists' sessions are read off the page as drawn, after a reload, so a streamed copy cannot pass. Before each reload the run polls the server until the answer lands; that wait is not graded. + +**Model:** n/a. kitchen-sink's scripted model answers the specialists and scripts best fit's evaluation (epic FIX-1592 D3). The goal is who ran, not what they said. Keyless: the server runs with `AI_GATEWAY_API_KEY` empty. **Run:** `PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/run.mts` -**Controls:** on the same command. The app honours each only under `KITCHEN_SINK_TEST_MODE=1`. +**Controls:** on the same command. The app honours each only under `KITCHEN_SINK_TEST_MODE=1` (`apps/kitchen-sink/lib/coordinator-control.ts`). -- `GOAL_CONTROL=name-only-notify`: the wake replaced by a line naming each member, and no seat run. Must FAIL at **support.accounts**, and at nothing else. -- `GOAL_CONTROL=no-route`: the mailbox's `routing:` lines taken off before the mailbox is bound, so every member hears every post. It is the **others** leg's control, which a fresh server could otherwise pass on empty lists. Must FAIL at **others**, and at nothing else. -- `GOAL_CONTROL=no-author-filter`: the wake's author filter dropped, so a post a seat wrote wakes the members too. It is leg c's control (FIX-1594); a post from the page carries no author, so here it must leave **every** leg green. The members the route passed over do run under it, on the specialist's answer landing as a line, but that line carries no token, and grading it is leg c's job: its red is `goals/kitchen-sink-talk/agent-replies-in-the-mailbox/`'s **woken-once** and `apps/kitchen-sink/test/mailbox-wake.test.ts`. +- `GOAL_CONTROL=no-delivery`: the coordinator registered with no flow that takes a delegated post, so nobody runs. Must FAIL at **support.accounts**, and at nothing else. +- `GOAL_CONTROL=no-route`: `support.help`'s best fit read as `everyone`, so every delegate hears every post. It is the **others** leg's control, which a fresh server could otherwise pass on empty lists. Must FAIL at **others**, and at nothing else. +- `GOAL_CONTROL=answers-go-on`: the coordinator read with `rounds: 1`, so an answer goes back out to the delegates. It is `agent-replies-in-the-mailbox`'s **woken-once** control; the answers carry no token here, so it must leave **every** leg green. ## Verdict log | Date | Commit | Model | Verdict | Notes | @@ -38,3 +40,8 @@ | 2026-09-27 | FIX-1611 branch on 315976e5f, uncommitted | scripted | FAIL (control) | **Re-pointed.** `GOAL_CONTROL=no-route`, new with the re-point: the **others** leg's own control, in place of the temporary `desk-clerk` wake the FIX-1590 blast-radius row used. Failed at **others** only: `support.devices`, `support.fsd` and `support.general` each held 1 conversation with the posts' tokens. **support.accounts** green. | | 2026-09-27 | FIX-1611 branch on 315976e5f, uncommitted | scripted | **PASS** (control) | **Re-pointed.** `GOAL_CONTROL=no-author-filter`: every leg green, as it must be. The three passed-over members each list 1 conversation under it, holding neither token: they ran on `support.accounts`' answer line, which is leg c's to grade. | | 2026-09-27 | FIX-1611 branch on 315976e5f, uncommitted | scripted | **PASS** | **Re-pointed.** Fresh production build, keyless. After a reload `support.accounts` lists one run of `support.help` holding both posts (`devuser in support.help: [route:support.accounts] [scenario:wake] wake-token-a… can someone look at the refund queue?` then the routed-turn line) with a `[reply:wake]` reply under each; `support.devices`, `support.fsd` and `support.general` list 0 conversations. | +| 2026-10-09 | `main` at 57d2c10e9 | scripted | FAIL | **Taken first, on clean `main`, before the conversion.** The previous check fails at setup: it waits for a rail row named `support.accounts`, and since FIX-1788 P4 the rail lists one `agent` copy and no row per worker. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Converted.** Fresh production build, keyless. After a reload, one session of `support.accounts` (`dsx_…`, listed in the rail as a run of the person's conversation with `support.help`) holds both posts as its turns, *devuser, through support.help: [route:support.accounts] [scenario:wake] wake-token-…*, a `[reply:wake]` reply under each; `support.devices`, `support.fsd` and `support.general` hold no session with either token. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-delivery`: **support.accounts** only — *0 of support.accounts's 0 sessions hold the first post*, and the second. **others** green. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-route`: **others** only — `support.devices`, `support.fsd` and `support.general` each hold 1 session with the posts' tokens. **support.accounts** green. | +| 2026-10-09 | same | scripted | PASS (expected) | `GOAL_CONTROL=answers-go-on`: every leg green, as the control's row says. | diff --git a/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/run.mts b/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/run.mts index 4de4b69cfc..35bc4c1564 100644 --- a/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/run.mts +++ b/goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/run.mts @@ -1,31 +1,32 @@ /** * Goal check: when a person posts to `support.help` from the kitchen-sink page, - * the one specialist the mailbox's route picks runs once on the post and no - * other member runs, and that run is there in the specialist's conversation + * the one specialist the coordinator's best fit picks runs once on the post and + * no other delegate runs, and that run is there in the specialist's session * after a reload. * * Real path, scripted model, out of CI. See goal.md for the contract. * * One real browser against the app's PRODUCTION build (built here, never - * assumed), on its scripted model, keyless. The scripted route picks the - * member a post names in `[route:]`. Two legs: + * assumed), on its scripted model, keyless. The scripted evaluation picks the + * delegate a post names in `[route:]`. Two legs: * * support.accounts post a line naming it, carrying a fresh token, from the - * mailbox's panel; reload; open every conversation the seat lists. - * Exactly one holds the token: the post heard once, as the seat's - * turn, with one reply carrying the wake marker under it. Then a - * second line, different text; reload; it is in that same - * conversation, heard once, answered once. - * others support.devices, support.fsd and support.general, the members the - * route did not pick, hold nothing with either token. + * coordinator's panel; reload; open every session the specialist + * has. Exactly one holds the token: the post heard once, as the + * specialist's turn, with one reply carrying the wake marker under it. + * Then a second line, different text; reload; it is in that same + * session, heard once, answered once. + * others support.devices, support.fsd and support.general, the delegates + * best fit did not pick, hold nothing with either token. * - * Everything graded is read off the page after a reload, from the seats' own - * conversations, so only a run the server kept can pass. + * Which sessions are a specialist's is read from the server, by the session's + * `workerId`, as an index; what each holds is read off the page after a + * reload, so only a run the server kept can pass. * * Run: PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/a-post-runs-each-member-agent-once/run.mts - * Controls: GOAL_CONTROL=name-only-notify (a line naming each member, no seat run: must FAIL at support.accounts, and nothing else) - * GOAL_CONTROL=no-route (the mailbox's `routing:` lines taken off, so every member hears: must FAIL at others, and nothing else) - * GOAL_CONTROL=no-author-filter (leg c's control: must leave every leg here green) + * Controls: GOAL_CONTROL=no-delivery (no flow takes a delegated post, so nobody runs: must FAIL at support.accounts, and nothing else) + * GOAL_CONTROL=no-route (best fit read as `everyone`, so every delegate hears: must FAIL at others, and nothing else) + * GOAL_CONTROL=answers-go-on (agent-replies-in-the-mailbox's control: must leave every leg here green) */ import { randomUUID } from "node:crypto"; import type { Page } from "playwright"; @@ -33,13 +34,14 @@ import { loadFixture, runGoal } from "../../lib/index.mts"; import { buildKitchenSink, conversation, - open, + coordinatorConversation, openShell, + openWorkerCopy, panel, - rail, readUntil, - row, + showCoordinator, startKitchenSink, + workerSessions, type KitchenSinkServer, } from "../../lib/kitchen-sink.mts"; import { launchChromium } from "../../lib/playwright.mts"; @@ -51,10 +53,10 @@ interface Seat { interface Fixture { port: number; - mailbox: Seat; + coordinator: Seat; agents: Seat[]; others: Seat[]; - /** The tag the scripted route reads to pick who answers. */ + /** The tag the scripted evaluation reads to pick who answers. */ route: string; marker: string; replyMarker: string; @@ -65,13 +67,13 @@ const CONTROL = process.env.GOAL_CONTROL ?? ""; /** The legs each control must redden, and only those. */ const EXPECTED: Record = { - "name-only-notify": fixture.agents.map((seat) => seat.id), - // Every member hears an unrouted post, so the members the route would have - // passed over run too. The routed specialist still answers. + "no-delivery": fixture.agents.map((seat) => seat.id), + // Every delegate hears a post handed to everyone, so the delegates best fit + // would have passed over run too. The picked specialist still answers. "no-route": ["others"], - // Leg c's control (a seat's own post wakes nobody). A post from the page has - // no author, so here it must change nothing. - "no-author-filter": [], + // agent-replies-in-the-mailbox's control (an answer wakes nobody). The + // answers here carry no token, so here it must change nothing. + "answers-go-on": [], }; if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); @@ -79,29 +81,26 @@ if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { type Conversation = { sessionId: string; runOf: string | null; messages: Array<{ role: string; text: string }> }; -/** Post a line to the mailbox from its panel, and wait until it shows. */ +/** Post a line to the coordinator from its panel, and wait until it shows. */ async function post(page: Page, origin: string, line: string): Promise { await openShell(page, origin); - await open(page, fixture.mailbox.kind); - await row(page, fixture.mailbox.id).click(); - await panel(page).getByLabel("Post to this mailbox").fill(line); + await showCoordinator(page, fixture.coordinator.id); + await panel(page).getByLabel("Post to this coordinator").fill(line); await panel(page).getByRole("button", { name: "Send" }).click(); await readUntil( - () => panel(page).getByTestId("mailbox-line").filter({ hasText: line }).count(), + () => panel(page).getByTestId("coordinator-line").filter({ hasText: line }).count(), (n) => n > 0, 10_000, ); } /** - * Let the routed specialist answer `token` before the reload. Not graded: a seat - * that never ran simply times out here and fails on the page below. + * Let the picked specialist answer `token` before the reload. Not graded: a + * specialist that never ran simply times out here and fails on the page below. */ async function letAgentsAnswer(page: Page, origin: string, token: string, replies: number): Promise { const answered = async (seat: string): Promise => { - const listed = await page.request.get(`${origin}/api/flows/sessions?flowId=${seat}&include=dispatch-runs&limit=100`); - const { sessions } = (await listed.json()) as { sessions: Array<{ id: string }> }; - for (const session of sessions) { + for (const session of await workerSessions(page, origin, seat, { runs: true })) { const state = await page.request.get(`${origin}/api/flows/sessions/${session.id}/state?include_items=true&item_types=message&limit=1000`); const text = await state.text(); if (text.includes(token) && text.split(fixture.replyMarker).length - 1 >= replies) return true; @@ -115,31 +114,24 @@ async function letAgentsAnswer(page: Page, origin: string, token: string, replie ); } -/** Reload, then open every conversation `seat` lists and read each as drawn. */ +/** Reload, then open every session `seat` has, from the worker copy in the rail, and read each as drawn. */ async function conversationsOf(page: Page, origin: string, seat: Seat): Promise { await page.reload(); await openShell(page, origin); - await open(page, seat.kind); - await open(page, seat.id); - const leaf = rail(page).locator(`ul[data-leaf="${seat.id}"]`); - await leaf.waitFor({ timeout: 15_000 }); - // The list is loaded once it shows a row or says it has none. - await readUntil( - async () => (await leaf.locator("[data-session-id]").count()) + (await leaf.getByText("No sessions yet").count()), - (n) => n > 0, - 10_000, - ); - const rows = await leaf.locator("[data-session-id]").evaluateAll((buttons) => - buttons.map((button) => ({ - sessionId: button.getAttribute("data-session-id") ?? "", - runOf: button.getAttribute("data-dispatch-run-of"), - })), - ); + const listed = await workerSessions(page, origin, seat.id, { runs: true }); + if (listed.length === 0) return []; + const leaf = await openWorkerCopy(page); const out: Conversation[] = []; - for (const listed of rows) { - await leaf.locator(`[data-session-id="${listed.sessionId}"]`).click(); - const messages = await readUntil(() => conversation(page), (ms) => ms.length > 0, 5_000); - out.push({ ...listed, messages }); + let last = ""; + for (const session of listed) { + const button = leaf.locator(`[data-session-id="${session.id}"]`); + await button.waitFor({ timeout: 10_000 }); + await button.click(); + await readUntil(() => button.getAttribute("aria-current"), (v) => v === "true", 5_000); + // The panel keeps the last session until this one replaces it. + const messages = await readUntil(() => conversation(page), (ms) => ms.length > 0 && JSON.stringify(ms) !== last, 5_000); + last = JSON.stringify(messages); + out.push({ sessionId: session.id, runOf: await button.getAttribute("data-dispatch-run-of"), messages }); } return out; } @@ -166,23 +158,23 @@ await runGoal(async () => { const origin = server.origin; const page = await browser.newPage({ viewport: { width: 1440, height: 900 } }); - /** One agent seat's conversations holding `token`, graded: one, heard once, one reply under it. */ + /** One agent's sessions holding `token`, graded: one, heard once, one reply under it. */ const gradeAgent = (seat: Seat, convs: Conversation[], token: string, label: string): Conversation | undefined => { const holding = convs.filter((c) => c.messages.some((m) => m.text.includes(token))); if (holding.length !== 1) { - fail(seat.id, `after the reload, ${holding.length} of ${seat.id}'s ${convs.length} conversations hold the ${label} post (want 1): it never ran on it, or ran in more than one`); + fail(seat.id, `after the reload, ${holding.length} of ${seat.id}'s ${convs.length} sessions hold the ${label} post (want 1): it never ran on it, or ran in more than one`); return undefined; } const [conv] = holding; const heard = conv!.messages.findIndex((m) => m.role === "user" && m.text.includes(token)); const heardCount = conv!.messages.filter((m) => m.role === "user" && m.text.includes(token)).length; if (heard === -1 || heardCount !== 1) { - fail(seat.id, `${seat.id}'s conversation holds the ${label} post as its turn ${heardCount} times (want 1); roles on screen: ${conv!.messages.map((m) => m.role).join(", ")}`); + fail(seat.id, `${seat.id}'s session holds the ${label} post as its turn ${heardCount} times (want 1); roles on screen: ${conv!.messages.map((m) => m.role).join(", ")}`); return undefined; } const next = conv!.messages[heard + 1]; if (next === undefined || next.role !== "assistant" || !next.text.includes(fixture.replyMarker)) { - fail(seat.id, `no reply carrying ${fixture.replyMarker} sits under the ${label} post in ${seat.id}'s conversation; under it: ${JSON.stringify(next ?? null)}`); + fail(seat.id, `no reply carrying ${fixture.replyMarker} sits under the ${label} post in ${seat.id}'s session; under it: ${JSON.stringify(next ?? null)}`); return undefined; } return conv; @@ -197,21 +189,22 @@ await runGoal(async () => { if (conv !== undefined) firstConversation.set(seat.id, conv); } - // ---- a second post, different text: the same conversation -------------- + // ---- a second post, different text: the same session ------------------ await post(page, origin, secondLine); await letAgentsAnswer(page, origin, secondToken, 2); + const helpConversation = await coordinatorConversation(page, origin, fixture.coordinator.id); for (const seat of fixture.agents) { const convs = await conversationsOf(page, origin, seat); const conv = gradeAgent(seat, convs, secondToken, "second"); const first = firstConversation.get(seat.id); if (conv === undefined || first === undefined) continue; if (conv.sessionId !== first.sessionId) { - fail(seat.id, `the second post landed in ${conv.sessionId}, not in the conversation the first one did (${first.sessionId})`); + fail(seat.id, `the second post landed in ${conv.sessionId}, not in the session the first one did (${first.sessionId})`); } else if (!conv.messages.some((m) => m.role === "user" && m.text.includes(firstToken))) { - fail(seat.id, `the conversation holding the second post no longer shows the first`); + fail(seat.id, `the session holding the second post no longer shows the first`); } else { evidence.push( - `${seat.id}: after a reload, one conversation ${conv.sessionId} (listed as a run of ${conv.runOf ?? "nothing"}) holds both posts as its turns, ` + + `${seat.id}: after a reload, one session ${conv.sessionId} (listed as a run of ${conv.runOf ?? "nothing"}, the person's conversation with ${fixture.coordinator.id} being ${helpConversation ?? "none"}) holds both posts as its turns, ` + `${JSON.stringify(conv.messages.find((m) => m.role === "user" && m.text.includes(firstToken))!.text)} with a ${fixture.replyMarker} reply under each`, ); } @@ -222,9 +215,9 @@ await runGoal(async () => { const convs = await conversationsOf(page, origin, seat); const holding = convs.filter((c) => c.messages.some((m) => m.text.includes(firstToken) || m.text.includes(secondToken))); if (holding.length > 0) { - fail("others", `${seat.id} holds ${holding.length} conversation(s) with the posts' tokens: a post ran a member the route did not pick`); + fail("others", `${seat.id} holds ${holding.length} session(s) with the posts' tokens: a post ran a delegate best fit did not pick`); } else { - evidence.push(`${seat.id}: none of its ${convs.length} conversations holds either token`); + evidence.push(`${seat.id}: none of its ${convs.length} sessions holds either token`); } } } finally { diff --git a/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/fixtures/input.json b/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/fixtures/input.json index 0eb7524f08..35d469217d 100644 --- a/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/fixtures/input.json +++ b/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/fixtures/input.json @@ -1,6 +1,6 @@ { "port": 3040, - "mailbox": { "kind": "mailbox", "id": "support.help" }, + "coordinator": { "kind": "coordinator", "id": "support.help" }, "replier": { "kind": "agent", "id": "support.devices" }, "others": [ { "kind": "agent", "id": "support.accounts" }, diff --git a/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/goal.md b/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/goal.md index b256122287..5b241cf684 100644 --- a/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/goal.md +++ b/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/goal.md @@ -1,29 +1,31 @@ # kitchen-sink-talk › an agent replies in the mailbox -**Issue:** FIX-1594 (VG of the spec's PLAN; the epic FIX-1592's leg c); re-pointed by FIX-1611 +**Issue:** FIX-1594 (VG of the spec's PLAN; the epic FIX-1592's leg c); re-pointed by FIX-1611; converted by FIX-1792 P1 -**Outcome:** A person posts to `support.help`, and the specialist the route picks, `support.devices`, answers in that mailbox under its own name through its `post-to-mailbox` tool. The line is still there after a reload, and wakes nobody. +**Outcome:** A person posts to `support.help`, and the specialist the coordinator's best fit picks, `support.devices`, answers in the person's conversation with `support.help` under its own name. The line is still there after a reload, and wakes nobody. -**Re-pointed (FIX-1611).** The folder name is kept so its verdict history stays in one place. Until FIX-1611 this read `support.otto` answering in `support.desk`, beside a second agent member. The team now has one routed mailbox, so the answering seat is the routed specialist and "wakes nobody" covers the three members the route passed over. +**Re-pointed (FIX-1611).** The folder name is kept so its verdict history stays in one place. Until FIX-1611 this read `support.otto` answering in `support.desk`, beside a second agent member. The team now has one routed help desk, so the answering seat is the routed specialist and "wakes nobody" covers the three it passed over. -**Input:** `fixtures/input.json`: the mailbox, the specialist a post routes to (`support.devices`), the three members it passes over (`support.accounts`, `support.fsd`, `support.general`), the `[route:]` tag the scripted route reads, the scenario marker the scripted model keys on (`[scenario:reply-in-mailbox]`) and the marker it writes on the specialist's line (`[reply:in-mailbox]`). Held-out: each run posts two lines, each with a fresh `reply-token-…`, and only those tokens are graded, so a line another run or test left in the shared mailbox can never be the one that passes. The second post, with different text, must pass the same way. +**Converted (FIX-1792 P1).** `support.help` is a coordinator worker file on `routing: best-fit`, no longer a mailbox. The specialist no longer posts its answer with a `post-to-mailbox` tool: its turn's reply is handed back to the coordinator, which draws it in the conversation under the specialist's name. An answer goes no further because the coordinator's `rounds:` is 0, not because of an author filter. The controls follow: `no-author-filter` becomes `answers-go-on`, and `post-without-author` (a server-side tool swap, which has no tool to swap now) becomes `no-author-name`, which takes the writer's name off each line the page is served. -**Signal:** one real browser against kitchen-sink's **production build** (built by the run, never assumed), served by `next start` on the scripted model, the in-memory store and **no model key**. Both posts go in from the mailbox's panel; everything graded is read off the page **after one reload**. +**Input:** `fixtures/input.json`: the coordinator, the specialist a post routes to (`support.devices`), the three delegates it passes over (`support.accounts`, `support.fsd`, `support.general`), the `[route:]` tag the scripted evaluation reads, the scenario marker the scripted model keys on (`[scenario:reply-in-mailbox]`) and the marker it writes on the specialist's answer (`[reply:in-mailbox]`). Held-out: each run posts two lines, each with a fresh `reply-token-…`, and only those tokens are graded, so a line another run or test left behind can never be the one that passes. The second post, with different text, must pass the same way. -- **line**: for each token, the `support.help` panel shows exactly one line carrying the token and `[reply:in-mailbox]`: the specialist's answer, kept by the mailbox. +**Signal:** one real browser against kitchen-sink's **production build** (built by the run, never assumed), served by `next start` on the scripted model, the in-memory store and **no model key**. Both posts go in from the coordinator's panel; everything graded is read off the page **after one reload**. + +- **line**: for each token, the conversation shows exactly one line carrying the token and `[reply:in-mailbox]`: the specialist's answer, kept in the conversation. - **author**: that line is labelled `support.devices`, not `devuser`. -- **woken-once**: `support.devices` lists one run of `support.help`, and in it each token appears in exactly one turn, the person's post (`devuser in support.help: [route:support.devices] [scenario:reply-in-mailbox] …`). `support.accounts`, `support.fsd` and `support.general` list no run of `support.help` holding either token. A second turn carrying the token, or another member's run, is the specialist's line waking a seat. +- **woken-once**: `support.devices` has one session for the conversation, and in it each token appears in exactly one turn, the person's post (`devuser, through support.help: [route:support.devices] [scenario:reply-in-mailbox] …`). `support.accounts`, `support.fsd` and `support.general` hold no session for the conversation with either token. A second turn carrying the token, or another delegate's session, is the specialist's answer waking a seat. -**Anti-game:** no assertion on the reply's wording past its marker, which the script wrote; none on the tool's return value, the route's record, a dispatch handle, a package test or a CLI run. Only this run's tokens count. The page is read after a reload, so only what the mailbox and the seats kept can pass. Before the reload the run polls the server until the line and the specialist's answer land, then waits 1.5s for a wrongly woken seat to run; none of that is graded. +**Anti-game:** no assertion on the reply's wording past its marker, which the script wrote; none on the routing record, a dispatch handle, a package test or a CLI run. Only this run's tokens count. The page is read after a reload, so only what the conversation and the specialists' sessions kept can pass. Which sessions are a specialist's is read from the server by the session's `workerId`, as an index; what each holds is read off the page. Before the reload the run polls the server until the line and the specialist's answer land, then waits 1.5s for a wrongly woken seat to run; none of that is graded. -**Model:** n/a. kitchen-sink's scripted model picks the route and answers the seat (epic FIX-1592 D3): a step calling `post-to-mailbox` with no text, so the real tool runs, then a text step. Keyless: the server runs with `AI_GATEWAY_API_KEY` empty. +**Model:** n/a. kitchen-sink's scripted model scripts best fit's evaluation and answers the specialist (epic FIX-1592 D3). Keyless: the server runs with `AI_GATEWAY_API_KEY` empty. **Run:** `PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/agent-replies-in-the-mailbox/run.mts` -**Controls:** on the same command. The app honours each only under `KITCHEN_SINK_TEST_MODE=1` (`apps/kitchen-sink/lib/goal-control.ts`). +**Controls:** on the same command. -- `GOAL_CONTROL=no-author-filter` (FIX-1590's): the wake's author filter dropped. The specialist's line is still signed, so the mailbox does not route it; it fans out to every member, and now wakes them. Must FAIL at **woken-once**, and at nothing else. -- `GOAL_CONTROL=post-without-author` (`apps/kitchen-sink/lib/mailbox-post-control.ts`): the tool swapped for one that posts no author. The line reads `devuser`, so it must FAIL at **author**. A line with no author is a person's post, so the route places it and a seat hears the specialist's own words: it must FAIL at **woken-once** too. At nothing else: **line** stays green. +- `GOAL_CONTROL=answers-go-on` (`apps/kitchen-sink/lib/coordinator-control.ts`, honoured only under `KITCHEN_SINK_TEST_MODE=1`): the coordinator read with `rounds: 1`, so each answer goes back out. Best fit has no route tag in the answer to read, so it lands on the fallback, `support.general`, handed the specialist's words, token and all. Must FAIL at **woken-once**, and at nothing else. +- `GOAL_CONTROL=no-author-name`: at the page's `fetch`, every session read, action stream and live session stream reaches the page with each message's `agentName` taken off, before and after the reload alike: what the page sees when the server keeps no writer on a line. The line is drawn under no specialist's name. Must FAIL at **author**, and at nothing else. ## Verdict log | Date | Commit | Model | Verdict | Notes | @@ -37,3 +39,7 @@ | 2026-09-27 | FIX-1611 branch on 315976e5f, uncommitted | scripted | FAIL (control) | **Re-pointed** at `support.help`, replier `support.devices` (routed by `[route:support.devices]`), the other three specialists as the members **woken-once** now also reads. **`GOAL_CONTROL=no-author-filter`, taken first.** Failed at **woken-once** only: the specialist's signed line fanned out unrouted and woke every member, so `support.devices` heard each token in 2 turns (the post, then its own line) and `support.accounts`, `support.fsd` and `support.general` each held a run with a token. **line** and **author** green. | | 2026-09-27 | FIX-1611 branch on 315976e5f, uncommitted | scripted | FAIL (control) | **Re-pointed.** `GOAL_CONTROL=post-without-author`: failed at **author** (both lines labelled `devuser`) and **woken-once** (`support.devices` heard each token twice: the mailbox held the unauthored line on the member already on the person's last post, so the route sent the specialist its own words), and at nothing else. **line** green. | | 2026-09-27 | FIX-1611 branch on 315976e5f, uncommitted | scripted | **PASS** | **Re-pointed.** Fresh production build, keyless. After one reload `support.help` shows one line per token, labelled `support.devices`, e.g. `[reply:in-mailbox] reply-token-a… Refunds post on Fridays.`; `support.devices` lists one run of `support.help`, each token heard once in the person's post; the other three specialists list 0 runs. | +| 2026-10-09 | `main` at 57d2c10e9 | scripted | FAIL | **Taken first, on clean `main`, before the conversion.** The previous check fails at setup: it waits for a rail row named for the specialist, and since FIX-1788 P4 the rail lists one `agent` copy and no row per worker. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Converted.** Fresh production build, keyless. After one reload the conversation with `support.help` shows one line per token, labelled `support.devices`: *[reply:in-mailbox] reply-token-… Refunds post on Fridays.* `support.devices` has one session for the conversation, each token heard once, in the person's post; `support.accounts`, `support.fsd` and `support.general` have no session for it. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=answers-go-on`: **woken-once** only — *support.general holds 1 session(s) for the conversation with a post's token*. **line** and **author** green. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-author-name`: **author** only — each reply line labelled `coordinator`, not `support.devices`. **line** and **woken-once** green. | diff --git a/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/run.mts b/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/run.mts index 92d9978bba..3046afd2be 100644 --- a/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/run.mts +++ b/goals/kitchen-sink-talk/agent-replies-in-the-mailbox/run.mts @@ -1,31 +1,31 @@ /** - * Goal check: a person posts to `support.help`, and the specialist the route - * picks, `support.devices`, answers in that mailbox under its own name through - * its `post-to-mailbox` tool. The line is still there after a reload, and - * wakes nobody. + * Goal check: a person posts to `support.help`, and the specialist the + * coordinator's best fit picks, `support.devices`, answers in the person's + * conversation under its own name. The line is still there after a reload, + * and wakes nobody. * * Real path, scripted model, out of CI. See goal.md for the contract. * * One real browser against the app's PRODUCTION build (built here, never - * assumed), on its scripted model, keyless. Two posts from the mailbox's - * panel, each naming the specialist in `[route:]` and carrying a fresh - * token, then one reload. Three legs, graded per post: + * assumed), on its scripted model, keyless. Two posts from the coordinator's + * panel, each naming `support.devices` (`[route:support.devices]`) and each + * with a fresh token; then one reload, after which everything is read off + * the page: * - * line the mailbox shows exactly one line carrying the post's token - * and the line marker: the specialist's answer, kept by the - * mailbox. - * author that line is labelled `support.devices`. - * woken-once `support.devices` lists one run of the mailbox, holding the - * token in exactly one turn: the person's post. The members the - * route passed over hold no run with the token. A second turn, - * or another member's run, is the specialist's line waking a seat. + * line for each token, the conversation shows exactly one line + * carrying it and the line marker: the specialist's answer. + * author that line is labelled `support.devices`, not `devuser`. + * woken-once `support.devices` has one session for the conversation, and in + * it each token is in exactly one turn, the person's post. + * support.accounts, support.fsd and support.general hold no + * session with either token. * - * Everything graded is read off the page after a reload, so only what the - * server kept can pass. + * Which sessions are a specialist's is read from the server, by the session's + * `workerId`, as an index; what each holds is read off the page. * * Run: PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/agent-replies-in-the-mailbox/run.mts - * Controls: GOAL_CONTROL=no-author-filter (must FAIL at woken-once, and nothing else) - * GOAL_CONTROL=post-without-author (must FAIL at author and woken-once, and nothing else) + * Controls: GOAL_CONTROL=answers-go-on (the coordinator read with `rounds: 1`: must FAIL at woken-once, and nothing else) + * GOAL_CONTROL=no-author-name (the page served each line without its writer's name: must FAIL at author, and nothing else) */ import { randomUUID } from "node:crypto"; import type { Page } from "playwright"; @@ -33,13 +33,14 @@ import { loadFixture, runGoal } from "../../lib/index.mts"; import { buildKitchenSink, conversation, - open, + coordinatorConversation, openShell, + openWorkerCopy, panel, - rail, readUntil, - row, + showCoordinator, startKitchenSink, + workerSessions, type KitchenSinkServer, } from "../../lib/kitchen-sink.mts"; import { launchChromium } from "../../lib/playwright.mts"; @@ -51,11 +52,11 @@ interface Seat { interface Fixture { port: number; - mailbox: Seat; + coordinator: Seat; replier: Seat; - /** The members the route passes over. */ + /** The delegates best fit passes over. */ others: Seat[]; - /** The tag the scripted route reads to pick who answers. */ + /** The tag the scripted evaluation reads to pick who answers. */ route: string; marker: string; lineMarker: string; @@ -66,85 +67,136 @@ const CONTROL = process.env.GOAL_CONTROL ?? ""; /** The legs each control must redden, and only those. */ const EXPECTED: Record = { - // The wake's author filter dropped: the specialist's line, still signed, is - // a seat's post, so it fans out unrouted and wakes every member. - "no-author-filter": ["woken-once"], - // The tool sends no author: the line reads as the principal, and a line with - // no author is a person's post, so the route places it and a seat hears the - // specialist's own words. - "post-without-author": ["author", "woken-once"], + // Each answer goes back out, never to its writer: best fit places it on the + // fallback, which is handed the specialist's words, token and all. + "answers-go-on": ["woken-once"], + // The page reads each line's writer off what the server keeps; served + // without it, the answer is drawn under no specialist's name. + "no-author-name": ["author"], }; if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); } -/** Open the mailbox's panel. */ -async function openMailbox(page: Page, origin: string): Promise { +/** + * `no-author-name`, at the page's `fetch`: every session read, action stream + * and live session stream reaches the page with each message's `agentName` + * taken off, before and after the reload alike. What the page sees when the + * server keeps no writer on a line. The server is untouched: the control + * graded here is that the page draws the writer the server kept. + * + * At the page's `fetch`, not with Playwright's routing: the live session + * stream never ends, and a routed response is read whole before the page gets + * any of it. Plain JavaScript in a string: a function handed to Playwright is + * compiled by tsx first, which adds helpers the page does not have. + */ +const STRIP_AUTHOR = `(() => { + const strip = (item) => { + if (item != null && item.type === "message" && "agentName" in item) delete item.agentName; + return item; + }; + const original = window.fetch.bind(window); + window.fetch = async (input, init) => { + const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, location.href); + const method = String((init && init.method) || (input instanceof Request ? input.method : "GET")).toUpperCase(); + const read = method === "GET" && /^\\/api\\/flows\\/sessions\\/[^/]+\\/(state|stream)$/.test(url.pathname); + const action = method === "POST" && /\\/actions\\/[^/]+$/.test(url.pathname); + const response = await original(input, init); + if (!read && !action) return response; + const type = response.headers.get("content-type") || ""; + if (type.includes("application/json")) { + const json = await response.json(); + if (json && Array.isArray(json.items)) json.items.forEach(strip); + return new Response(JSON.stringify(json), { status: response.status, statusText: response.statusText, headers: response.headers }); + } + if (!type.includes("text/event-stream") || response.body === null) return response; + const decoder = new TextDecoder(); + const encoder = new TextEncoder(); + let buffer = ""; + const rewrite = (frame) => frame.split("\\n").map((line) => { + if (!line.startsWith("data:")) return line; + try { + const parsed = JSON.parse(line.slice(5).trim()); + if (parsed && parsed.item) strip(parsed.item); + return "data: " + JSON.stringify(parsed); + } catch { return line; } + }).join("\\n"); + const filtered = response.body.pipeThrough(new TransformStream({ + transform(chunk, controller) { + buffer += decoder.decode(chunk, { stream: true }); + for (let at = buffer.indexOf("\\n\\n"); at !== -1; at = buffer.indexOf("\\n\\n")) { + const frame = buffer.slice(0, at); + buffer = buffer.slice(at + 2); + controller.enqueue(encoder.encode(rewrite(frame) + "\\n\\n")); + } + }, + flush(controller) { if (buffer !== "") controller.enqueue(encoder.encode(rewrite(buffer))); }, + })); + return new Response(filtered, { status: response.status, statusText: response.statusText, headers: response.headers }); + }; +})();`; + +/** Open the person's conversation with the coordinator in the panel. */ +async function openHelpDesk(page: Page, origin: string): Promise { await openShell(page, origin); - await open(page, fixture.mailbox.kind); - await row(page, fixture.mailbox.id).click(); - await panel(page).getByTestId("mailbox-transcript").waitFor({ timeout: 15_000 }); + await showCoordinator(page, fixture.coordinator.id); } -/** Post a line from the mailbox's panel, and wait until it shows. */ +/** Post a line from the conversation's panel, and wait until it shows. */ async function post(page: Page, line: string): Promise { - await panel(page).getByLabel("Post to this mailbox").fill(line); + await panel(page).getByLabel("Post to this coordinator").fill(line); await panel(page).getByRole("button", { name: "Send" }).click(); await readUntil( - () => panel(page).getByTestId("mailbox-line").filter({ hasText: line }).count(), + () => panel(page).getByTestId("coordinator-line").filter({ hasText: line }).count(), (n) => n > 0, 10_000, ); } /** - * Let the specialist's line land and its answer finish before the reload, then - * give a wrongly woken seat time to run. Not graded: whatever did not happen fails - * on the page below. + * Let the specialist's line land and its turn finish before the reload, then + * give a wrongly woken delegate time to run. Not graded: whatever did not + * happen fails on the page below. */ async function settle(page: Page, origin: string, token: string): Promise { - const mailboxHolds = async () => { - const res = await page.request.get( - `${origin}/api/flows/sessions/${fixture.mailbox.id}/state?include_items=true&item_types=component&limit=1000`, - ); + const conversationHolds = async () => { + const id = await coordinatorConversation(page, origin, fixture.coordinator.id); + if (id === undefined) return false; + const res = await page.request.get(`${origin}/api/flows/sessions/${id}/state?include_items=true&item_types=message&limit=1000`); const text = await res.text(); return text.includes(fixture.lineMarker) && text.includes(token); }; const answered = async (seat: string) => { - const listed = await page.request.get(`${origin}/api/flows/sessions?flowId=${seat}&include=dispatch-runs&limit=100`); - const { sessions } = (await listed.json()) as { sessions: Array<{ id: string }> }; - for (const session of sessions) { + for (const session of await workerSessions(page, origin, seat, { runs: true })) { const state = await page.request.get(`${origin}/api/flows/sessions/${session.id}/state?include_items=true&item_types=message&limit=1000`); if ((await state.text()).includes(token)) return true; } return false; }; await readUntil( - async () => (await mailboxHolds()) && (await answered(fixture.replier.id)), + async () => (await conversationHolds()) && (await answered(fixture.replier.id)), (done) => done, 20_000, ); await page.waitForTimeout(1_500); } -/** A seat's runs of the mailbox, each read as drawn. The page is already reloaded. */ -async function mailboxRunsOf(page: Page, origin: string, seat: Seat): Promise>> { +/** A delegate's sessions for the person's conversation, each read as drawn. The page is already reloaded. */ +async function delegateRunsOf(page: Page, origin: string, seat: Seat): Promise>> { await openShell(page, origin); - await open(page, seat.kind); - await open(page, seat.id); - const leaf = rail(page).locator(`ul[data-leaf="${seat.id}"]`); - await leaf.waitFor({ timeout: 15_000 }); - await readUntil( - async () => (await leaf.locator("[data-session-id]").count()) + (await leaf.getByText("No sessions yet").count()), - (n) => n > 0, - 10_000, - ); - const runs = leaf.locator(`[data-dispatch-run-of="${fixture.mailbox.id}"]`); - const ids = await runs.evaluateAll((buttons) => buttons.map((b) => b.getAttribute("data-session-id") ?? "")); + const parent = await coordinatorConversation(page, origin, fixture.coordinator.id); + const runs = (await workerSessions(page, origin, seat.id, { runs: true })).filter((s) => s.parentSessionId === parent); + if (runs.length === 0) return []; + const leaf = await openWorkerCopy(page); const out: Array> = []; - for (const id of ids) { - await leaf.locator(`[data-session-id="${id}"]`).click(); - out.push(await readUntil(() => conversation(page), (ms) => ms.length > 0, 5_000)); + let last = ""; + for (const { id } of runs) { + const button = leaf.locator(`[data-session-id="${id}"]`); + await button.waitFor({ timeout: 10_000 }); + await button.click(); + const messages = await readUntil(() => conversation(page), (ms) => ms.length > 0 && JSON.stringify(ms) !== last, 5_000); + last = JSON.stringify(messages); + out.push(messages); } return out; } @@ -171,23 +223,24 @@ await runGoal(async () => { server = await startKitchenSink(fixture.port, { AI_GATEWAY_API_KEY: "" }); const origin = server.origin; const page = await browser.newPage({ viewport: { width: 1440, height: 900 } }); + if (CONTROL === "no-author-name") await page.addInitScript({ content: STRIP_AUTHOR }); - await openMailbox(page, origin); + await openHelpDesk(page, origin); for (const [i, line] of lines.entries()) { await post(page, line); await settle(page, origin, tokens[i]!); } - // ---- after one reload, the mailbox as drawn ------------------------------ + // ---- after one reload, the conversation as drawn ------------------------ await page.reload(); - await openMailbox(page, origin); + await openHelpDesk(page, origin); const drawn = await readUntil( () => panel(page) - .getByTestId("mailbox-line") + .getByTestId("coordinator-line") .evaluateAll((els) => els.map((el) => ({ - label: el.querySelector('[data-testid="mailbox-line-label"]')?.textContent ?? "", + label: el.querySelector('[data-testid="coordinator-line-label"]')?.textContent ?? "", text: el.textContent ?? "", })), ), @@ -197,20 +250,20 @@ await runGoal(async () => { for (const token of tokens) { const replies = drawn.filter((l) => l.text.includes(token) && l.text.includes(fixture.lineMarker)); if (replies.length !== 1) { - fail("line", `after the reload, ${fixture.mailbox.id} shows ${replies.length} lines carrying ${token} and ${fixture.lineMarker} (want 1)`); + fail("line", `after the reload, the conversation with ${fixture.coordinator.id} shows ${replies.length} lines carrying ${token} and ${fixture.lineMarker} (want 1)`); continue; } if (replies[0]!.label !== fixture.replier.id) { fail("author", `the reply line for ${token} is labelled "${replies[0]!.label}", not ${fixture.replier.id}`); continue; } - evidence.push(`${fixture.mailbox.id}: one line for ${token}, labelled ${replies[0]!.label}: ${JSON.stringify(replies[0]!.text)}`); + evidence.push(`the conversation with ${fixture.coordinator.id}: one line for ${token}, labelled ${replies[0]!.label}: ${JSON.stringify(replies[0]!.text)}`); } // ---- the specialist heard each post once, and nobody heard its line ---- - const runs = await mailboxRunsOf(page, origin, fixture.replier); + const runs = await delegateRunsOf(page, origin, fixture.replier); if (runs.length !== 1) { - fail("woken-once", `${fixture.replier.id} lists ${runs.length} runs of ${fixture.mailbox.id} (want 1)`); + fail("woken-once", `${fixture.replier.id} has ${runs.length} sessions for the conversation with ${fixture.coordinator.id} (want 1)`); } else { let once = true; for (const token of tokens) { @@ -223,15 +276,15 @@ await runGoal(async () => { ); } } - if (once) evidence.push(`${fixture.replier.id}: one run of ${fixture.mailbox.id}, each token heard once, in the person's post`); + if (once) evidence.push(`${fixture.replier.id}: one session for the conversation, each token heard once, in the person's post`); } for (const seat of fixture.others) { - const theirs = await mailboxRunsOf(page, origin, seat); + const theirs = await delegateRunsOf(page, origin, seat); const holding = theirs.filter((ms) => ms.some((m) => tokens.some((t) => m.text.includes(t)))); if (holding.length > 0) { - fail("woken-once", `${seat.id} holds ${holding.length} run(s) of ${fixture.mailbox.id} with a post's token: the specialist's line woke it`); + fail("woken-once", `${seat.id} holds ${holding.length} session(s) for the conversation with a post's token: the specialist's line woke it`); } else { - evidence.push(`${seat.id}: none of its ${theirs.length} runs of ${fixture.mailbox.id} holds either token`); + evidence.push(`${seat.id}: none of its ${theirs.length} sessions for the conversation holds either token`); } } } finally { diff --git a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/fixtures/input.json b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/fixtures/input.json index 578cccb494..221cb55aa5 100644 --- a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/fixtures/input.json +++ b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/fixtures/input.json @@ -1,7 +1,7 @@ { "port": 3038, - "mailbox": { "kind": "mailbox", "id": "support.help", "board": "escalations" }, - "seatKind": "agent", + "coordinator": { "kind": "coordinator", "id": "support.help", "description": "Ask the support team anything." }, + "workerKind": "agent", "specialists": [ { "id": "support.devices", "description": "Printers, laptops, phones, wifi and anything else with a power button." }, { "id": "support.accounts", "description": "Sign-in, passwords, billing, refunds and subscriptions." }, @@ -10,14 +10,11 @@ ], "seat": "support.devices", "answer": { "marker": "[scenario:wake]", "replyMarker": "[reply:wake]" }, - "file": { "marker": "[scenario:needs-a-person]", "replyMarker": "[reply:escalated]" }, "live": { - "needsAPerson": [ + "posts": [ "My laptop battery is swelling and the case has started to split. I'm scared to keep using it and need someone on your team to arrange a replacement today.", "I've been charged three times this month for the same subscription and my bank says only you can reverse it. Please have a person look at my account and refund the extra charges.", - "Someone else signed in to my account and changed my email and password, so I'm locked out. I need a person to verify it's me and lock the account down." - ], - "answerable": [ + "Someone else signed in to my account and changed my email and password, so I'm locked out. I need a person to verify it's me and lock the account down.", "How do I reconnect my laptop to the office wifi after the password changed?", "Where on the billing page do I find my past invoices?" ] diff --git a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md index 48f1e49fdb..60a2252666 100644 --- a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md +++ b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md @@ -1,31 +1,37 @@ # kitchen-sink-talk › it answers a clerk note or files it -**Issue:** FIX-1611 (VG of the spec's PLAN), re-pointed from FIX-1589 (the epic FIX-1592's leg a). The path is kept: retained specs cite it. +**Issue:** FIX-1611 (VG of the spec's PLAN), re-pointed from FIX-1589 (the epic FIX-1592's leg a); rewritten by FIX-1792 P1. The path is kept: retained specs cite it. -**Outcome:** A person who opens kitchen-sink finds one support mailbox and four specialists named for what they handle. A question gets one specialist's answer in the mailbox; a case that needs a person is filed onto `escalations` and the specialist says so. Nobody gets their own words back, and after a reload the answer is still under the specialist's name and a filed case is a row in the team panel. +**Outcome:** A person who opens kitchen-sink finds one support coordinator and four specialists named for what they handle. A question posted to the coordinator gets one specialist's answer, under the specialist's name, and nobody gets their own words back. After a reload the answer is still there. -**Input:** `fixtures/input.json`: the mailbox and its board, the one seat kind, the four specialists with their `description:` lines, the specialist both legs aim at (`support.devices`), the scenario markers the scripted model keys on and the reply markers each writes, and the live leg's five posts. Held-out: each post carries a fresh token per run, and only that token is graded, so a line or row another run left behind can never be the one that passes. `GOAL_SEAT` aims both legs at another specialist; a correct build still passes. +**Rewritten (FIX-1792 P1).** `support.help` is a coordinator worker file on `routing: best-fit`, and the escalation feature is removed, not replaced (FIX-1792 BR-14; product owner, 2026-10-06): no specialist files a case, and there is no `escalations` board. So the filing half of this goal goes, with its control. What went, and why: -**Signal:** one real browser against kitchen-sink's **production build** (built by the run, never assumed), served by `next start` on the scripted model, the in-memory store and **no model key**. Each post is routed with `[route:]`, which the scripted route honours. Everything graded is read off the page as drawn. +- **file:line** and **file:row**: nothing files a case, so there is no "filed" line to read and no row to list. +- **warning**: the boot warned that `escalations` was unattended. With no board there is nothing to warn about. +- `GOAL_CONTROL=no-filing`: the stand-in for a tool that is gone. +- The live leg's filing half: its five posts now check one answer each, by a specialist. +- `main`'s roster red state (three mailboxes on two kinds) is history: the roster leg now reads the coordinator and the roster panel. -- **roster**: the rail's Mailboxes section lists one kind, `mailbox`, holding one mailbox, `support.help`; its Seats section lists one kind, `agent`, holding exactly the four specialists. Opened, each shows the kind `agent` and its `description:` as the fixture has it. No "Hire another" anywhere on the page. -- **answer**: post a question with `[scenario:wake]` and a fresh token; reload. The mailbox shows exactly one line after the post, labelled with the seat, carrying `[reply:wake]` and not the token. -- **file**: post a case with `[scenario:needs-a-person]` and a fresh token; reload. Graded as two assertions: **file:line**, exactly one line after the post, labelled with the seat, carrying `[reply:escalated]`; **file:row**, the team panel's `escalations` board shows exactly one row carrying the token. -- **warning**: the boot warns that `support.help`'s `escalations` is unattended, and warns about no other board (filing is not draining). +Since FIX-1788 the rail lists each flow, not each worker, so "four specialists named for what they handle" is read off the roster panel, where each worker shows its flow and its `description:`. -**Anti-game:** no assertion on the generated map, the ledger or the request log: the rail, the mailbox and the board column as drawn. A "filed" line alone never passes: the row is its own assertion. The reply must not carry the post's token, which is what an echo cannot avoid. Only this run's tokens count. Waiting before the reload polls the server until the seat's line lands and the row is written; none of that is graded. +**Input:** `fixtures/input.json`: the coordinator and its `description:`, the one specialist flow, the four specialists with their `description:` lines, the specialist the answer leg aims at (`support.devices`), the scenario marker the scripted model keys on and the reply marker it writes, and the live leg's five posts. Held-out: the post carries a fresh token per run, and only that token is graded, so a line another run left behind can never be the one that passes. `GOAL_SEAT` aims the answer leg at another specialist; a correct build still passes. -**Model:** n/a for the graded legs: kitchen-sink's scripted model answers and routes (epic FIX-1592 D3). The goal is which path ran and what was kept, not what the reply says. Keyless: the server runs with `AI_GATEWAY_API_KEY` empty. +**Signal:** one real browser against kitchen-sink's **production build** (built by the run, never assumed), served by `next start` on the scripted model, the in-memory store and **no model key**. The post is routed with `[route:]`, which the scripted evaluation honours. Everything graded is read off the page as drawn. + +- **roster**: the rail's Coordinators section lists one kind, `coordinator`, opening on the person's conversation with `support.help`; its Workers section lists one kind, `agent`. The roster panel lists exactly `support.help` on `coordinator` and the four specialists on `agent`, each with its `description:` as the fixture has it. No "Hire another" anywhere on the page. +- **answer**: post a question with `[scenario:wake]` and a fresh token; reload. The conversation shows exactly one line after the post, labelled with the specialist, carrying `[reply:wake]` and not the token. + +**Anti-game:** no assertion on the generated map, the ledger, the routing record or the request log: the rail, the roster panel and the conversation as drawn. The reply must not carry the post's token, which is what an echo cannot avoid. Only this run's token counts. Waiting before the reload polls the server until the specialist's line lands; that is not graded. + +**Model:** n/a for the graded legs: kitchen-sink's scripted model answers and scripts best fit's evaluation (epic FIX-1592 D3). The goal is which path ran and what was kept, not what the reply says. Keyless: the server runs with `AI_GATEWAY_API_KEY` empty. **Run:** `PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/run.mts` -**Live:** `GOAL_LIVE=1` on the same command, with `AI_GATEWAY_API_KEY`. After the scripted legs, the same build served out of test mode, on the app's own models and route. Three posts that plainly need a person must each add one row to `escalations`; two that do not must add none; each must get one answer line. +**Live:** `GOAL_LIVE=1` on the same command, with `AI_GATEWAY_API_KEY`. After the scripted legs, the same build served out of test mode, on the app's own models and best fit. Each of five posts must get exactly one answer line, by one of the four specialists. -**Controls:** on the same command. The app honours each only under `KITCHEN_SINK_TEST_MODE=1` (`apps/kitchen-sink/lib/goal-control.ts`). +**Controls:** on the same command. The app honours it only under `KITCHEN_SINK_TEST_MODE=1` (`apps/kitchen-sink/lib/landing-control.ts`). -- `GOAL_CONTROL=no-landing` (`apps/kitchen-sink/lib/mailbox-landing-control.ts`): the agent kind swapped for one that hears posts and answers with the seat's tools, and lands no text answer. A text answer no longer lands, and the specialist's "filed" is a text answer too, so it must FAIL at **answer** and **file:line**. The row still lands, so **file:row** stays green: nothing else. -- `GOAL_CONTROL=no-filing` (`apps/kitchen-sink/lib/escalate-control.ts`): `escalate` swapped for a stand-in that says it filed and files nothing. Must FAIL at **file:row** only: the line still says it filed. -- Today's `main`, before this issue, must FAIL at **roster**: it lists three mailboxes on two kinds, and six seats on three kinds. Its "Hire another" is never reached there: the button draws only on an opened seat, and none of the four specialists exists to open. That assertion's own red state is the button put back by hand, recorded below. +- `GOAL_CONTROL=no-landing`: the `agent` flow swapped for one that hears the delegated post and answers in the specialist's own session, and hands nothing back. The answer never reaches the conversation, so it must FAIL at **answer**. **roster** stays green: nothing else. ## Verdict log | Date | Commit | Model | Verdict | Notes | @@ -46,3 +52,6 @@ | 2026-09-28 | `fix/fix-1623` (FIX-1624) on 9a55b79e4, uncommitted (control: `no-landing`) | scripted | FAIL (expected) | Re-taken after the stand-in began carrying `post-to-mailbox` and drawing its answers as the real kind does. **answer** and **file:line** only: after the reload no line answers either token. **file:row**, **roster** and **warning** green. | | 2026-09-28 | same | scripted | **PASS** | roster as above; after a reload one line by `support.devices`, `[reply:wake] Heard it in the mailbox.`; the filed line and the `escalations` row as above. | | 2026-09-28 | FIX-1622 branch at 91a5afdc4 (the escalations board is a live list read through `support.help`) | scripted | **PASS** | Neighbour re-run. roster as above; after a reload one line by `support.devices`, `[reply:wake] Heard it in the mailbox.`; the line reads `[reply:escalated] Filed onto escalations.`, and after a reload the board's `li[data-task-id]` shows `Needs a person: case-token-…` with its status `pending`. The boot warns about `support.help.escalations` and no other board. | +| 2026-10-09 | `main` at 57d2c10e9 | scripted | FAIL | **Rewritten for FIX-1792 P1; the check before the rewrite taken first, on clean `main`.** It failed at **roster**: the rail's Seats section lists the kinds `[]` (since FIX-1788 P4 the section is Workers, and holds one `agent` copy). | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Rewritten.** Fresh production build, keyless. **roster:** the rail lists `coordinator`, opening on `["support.help"]`, and `agent`; the roster panel lists `support.help` (coordinator) *Ask the support team anything.* and the four specialists (agent), each with its description as the fixture has it; no "Hire another". **answer:** after a reload, one line answering the post, by `support.devices`: *[reply:wake] Heard it.*, without the token. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-landing`: **answer** only — after the reload no line answers the token. **roster** green. | diff --git a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/run.mts b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/run.mts index c5820d6765..3f5e392cd7 100644 --- a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/run.mts +++ b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/run.mts @@ -1,35 +1,28 @@ /** - * Goal check: a person who opens kitchen-sink finds one support mailbox and - * four specialists named for what they handle. A question gets one - * specialist's answer in the mailbox; a case that needs a person is filed onto - * `escalations` and the specialist says so. + * Goal check: a person who opens kitchen-sink finds one support coordinator + * and four specialists named for what they handle, and a question posted to + * the coordinator gets one specialist's answer, under the specialist's name. * * Real path, scripted model, out of CI. See goal.md for the contract. * - * Three legs, in one real browser against the app's PRODUCTION build (built + * Two legs, in one real browser against the app's PRODUCTION build (built * here, never assumed), on its scripted model, keyless: * - * roster the rail lists one mailbox and the four specialists, each of the - * one seat kind and with its description; no other mailbox, seat or - * kind, and no "Hire another" anywhere. + * roster the rail's Coordinators section lists one kind, `coordinator`, + * opening on the person's conversation with `support.help`; its + * Workers section lists one kind, `agent`. The roster panel lists + * exactly `support.help` on `coordinator` and the four specialists + * on `agent`, each with its description; no "Hire another" anywhere. * answer post a question routed to the seat, with a fresh token; after a - * reload the mailbox shows one line under the seat's name answering - * it, carrying the answer marker and not the token. - * file post a case that needs a person; after a reload the mailbox shows - * one line under the seat's name saying it filed (`file:line`), and - * the team panel's board shows exactly one row carrying the token - * (`file:row`). + * reload the conversation shows one line under the seat's name + * answering it, carrying the answer marker and not the token. * - * And the boot still warns that `escalations` is unattended, as the only - * such board (`warning`). - * - * `GOAL_LIVE=1`, with a key: the same build on real models. Three posts that - * plainly need a person are each filed once, two that don't file nothing, and - * each gets one answer line. + * `GOAL_LIVE=1`, with a key: the same build on real models and the app's own + * route. Each of five posts gets exactly one answer line, under one of the + * four specialists' names. * * Run: PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/run.mts - * Controls: GOAL_CONTROL=no-landing (must FAIL at answer and file:line, and nothing else) - * GOAL_CONTROL=no-filing (must FAIL at file:row, and nothing else) + * Controls: GOAL_CONTROL=no-landing (must FAIL at answer, and nothing else) * Held-out: GOAL_SEAT= * Live: GOAL_LIVE=1 with AI_GATEWAY_API_KEY */ @@ -38,12 +31,13 @@ import type { Page } from "playwright"; import { loadFixture, runGoal } from "../../lib/index.mts"; import { buildKitchenSink, + coordinatorConversation, open, openShell as openShellAt, panel, rail, readUntil, - row, + showCoordinator, startKitchenSink, type KitchenSinkServer, } from "../../lib/kitchen-sink.mts"; @@ -56,32 +50,27 @@ interface Specialist { interface Fixture { port: number; - mailbox: { kind: string; id: string; board: string }; - seatKind: string; + coordinator: { kind: string; id: string; description: string }; + workerKind: string; specialists: Specialist[]; seat: string; answer: { marker: string; replyMarker: string }; - file: { marker: string; replyMarker: string }; - live: { needsAPerson: string[]; answerable: string[] }; + live: { posts: string[] }; } const fixture = loadFixture(import.meta.url); const SEAT = process.env.GOAL_SEAT ?? fixture.seat; const CONTROL = process.env.GOAL_CONTROL ?? ""; const LIVE = process.env.GOAL_LIVE === "1"; -const MAILBOX = fixture.mailbox.id; +const COORDINATOR = fixture.coordinator.id; +/** The name the page draws the person's own lines under. */ +const PERSON = "devuser"; -/** - * The assertions each control must redden, and only those. `answer`, `roster` - * and `warning` are legs; the file leg is two assertions, graded apart: - * `file:line` (the seat's line says it filed) and `file:row` (the row). - */ +/** The legs each control must redden, and only those. */ const EXPECTED: Record = { - // A text answer no longer lands: the answer, and the line that says "filed". - // The row still lands, so the filing itself is not what the leg measures. - "no-landing": ["answer", "file:line"], - // The tool says it filed and files nothing: the row only. - "no-filing": ["file:row"], + // The specialist hears the post and answers, and its answer never comes + // back to the conversation. + "no-landing": ["answer"], }; if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); @@ -96,31 +85,35 @@ if (!fixture.specialists.some((s) => s.id === SEAT)) { type Drawn = Array<{ label: string; text: string }>; -/** Open the mailbox's panel. */ -async function openMailbox(page: Page, origin: string): Promise { +/** Open the person's conversation with the coordinator in the panel. */ +async function openHelpDesk(page: Page, origin: string): Promise { await openShellAt(page, origin); - await open(page, fixture.mailbox.kind); - await row(page, MAILBOX).click(); - await panel(page).getByTestId("mailbox-transcript").waitFor({ timeout: 15_000 }); + await showCoordinator(page, COORDINATOR); } -/** Post a line from the mailbox's panel, and wait until it shows. */ +/** Post a line from the conversation's panel, and wait until it shows. */ async function post(page: Page, line: string): Promise { - await panel(page).getByLabel("Post to this mailbox").fill(line); + await panel(page).getByLabel("Post to this coordinator").fill(line); await panel(page).getByRole("button", { name: "Send" }).click(); - await readUntil(() => panel(page).getByTestId("mailbox-line").filter({ hasText: line }).count(), (n) => n > 0, 10_000); + await readUntil(() => panel(page).getByTestId("coordinator-line").filter({ hasText: line }).count(), (n) => n > 0, 10_000); } -type Line = { author?: string; body: string }; - -/** The mailbox's lines as the server keeps them. Not graded: used only to wait. */ -async function keptLines(page: Page, origin: string): Promise { - const res = await page.request.get(`${origin}/api/flows/sessions/${MAILBOX}/state?include_items=true&item_types=component&limit=1000`); - const items = ((await res.json()) as { items?: Array<{ component?: string; data?: Line }> }).items ?? []; - return items.filter((item) => item.component === "mailbox-post").map((item) => item.data!); +type Kept = { author?: string; body: string }; + +/** The conversation's lines as the server keeps them. Not graded: used only to wait. */ +async function keptLines(page: Page, origin: string): Promise { + const id = await coordinatorConversation(page, origin, COORDINATOR); + if (id === undefined) return []; + const res = await page.request.get(`${origin}/api/flows/sessions/${id}/state?include_items=true&item_types=message&limit=1000`); + const items = + ((await res.json()) as { items?: Array<{ role?: string; agentName?: string; content?: Array<{ text?: string }> }> }).items ?? []; + return items.map((item) => ({ + ...(item.agentName === undefined ? {} : { author: item.agentName }), + body: (item.content ?? []).map((part) => part.text ?? "").join(""), + })); } -/** Wait until a seat's line follows the person's line carrying `mark`, or the time is up. Not graded. */ +/** Wait until a specialist's line follows the person's line carrying `mark`, or the time is up. Not graded. */ async function waitAnswered(page: Page, origin: string, mark: string, ms: number): Promise { await readUntil( () => keptLines(page, origin), @@ -133,7 +126,7 @@ async function waitAnswered(page: Page, origin: string, mark: string, ms: number } /** - * Reload, reopen the mailbox, and read it as drawn. + * Reload, reopen the conversation, and read it as drawn. * * The transcript mounts before its lines load, so this waits until the * person's lines carrying `marks` are drawn: every grade counts from them. @@ -142,38 +135,31 @@ async function waitAnswered(page: Page, origin: string, mark: string, ms: number */ async function drawnAfterReload(page: Page, origin: string, marks: string[]): Promise { await page.reload(); - await openMailbox(page, origin); + await openHelpDesk(page, origin); return await readUntil( () => panel(page) - .getByTestId("mailbox-line") + .getByTestId("coordinator-line") .evaluateAll((els) => els.map((el) => ({ - label: el.querySelector('[data-testid="mailbox-line-label"]')?.textContent ?? "", - text: el.textContent ?? "", + label: el.querySelector('[data-testid="coordinator-line-label"]')?.textContent ?? "", + text: el.querySelector('[data-testid="coordinator-line-body"]')?.textContent ?? "", })), ), - (drawn) => marks.every((mark) => drawn.some((l) => l.label === "devuser" && l.text.includes(mark))), + (drawn) => marks.every((mark) => drawn.some((l) => l.label === PERSON && l.text.includes(mark))), 10_000, ); } /** The lines drawn after the person's line carrying `mark`, up to the person's next line. */ function answersTo(drawn: Drawn, mark: string): Drawn | undefined { - const at = drawn.findIndex((l) => l.label === "devuser" && l.text.includes(mark)); + const at = drawn.findIndex((l) => l.label === PERSON && l.text.includes(mark)); if (at === -1) return undefined; const rest = drawn.slice(at + 1); - const next = rest.findIndex((l) => l.label === "devuser"); + const next = rest.findIndex((l) => l.label === PERSON); return next === -1 ? rest : rest.slice(0, next); } -/** The rows the team panel draws on the mailbox's board: each row's text. */ -const boardRows = (page: Page) => - page - .getByTestId(`board-${MAILBOX}.${fixture.mailbox.board}`) - .locator("li[data-task-id]") - .evaluateAll((rows) => rows.map((r) => r.textContent ?? "")); - /** A rail section's kind rows, by name. */ const sectionKinds = (page: Page, section: string) => rail(page) @@ -181,77 +167,79 @@ const sectionKinds = (page: Page, section: string) => .locator("button[data-kind]") .evaluateAll((bs) => bs.map((b) => b.getAttribute("data-kind") ?? "")); +/** The roster panel as drawn: each worker's id, its flow and its description. */ +const rosterRows = (page: Page) => + page + .locator('[data-testid="roster"]:visible [data-testid="roster-worker"]') + .evaluateAll((rows) => + rows.map((row) => ({ + id: row.getAttribute("data-worker-id") ?? "", + flow: row.querySelector(":scope > div > span:nth-of-type(2)")?.textContent ?? "", + description: row.querySelector(":scope > p:not([role])")?.textContent ?? "(none drawn)", + })), + ); + // --------------------------------------------------------------------------- // The legs // --------------------------------------------------------------------------- type Fail = (leg: string, line: string) => void; -/** roster: the rail as drawn. */ -async function gradeRoster(page: Page, origin: string, fail: Fail, evidence: string[]): Promise { +/** roster: the rail and the roster panel as drawn. */ +async function gradeRoster(page: Page, origin: string, failLeg: Fail, evidence: string[]): Promise { await openShellAt(page, origin); - const mailboxKinds = await sectionKinds(page, "Mailboxes"); - if (JSON.stringify(mailboxKinds) !== JSON.stringify([fixture.mailbox.kind])) { - fail("roster", `the rail's Mailboxes section lists the kinds ${JSON.stringify(mailboxKinds)} (want ["${fixture.mailbox.kind}"])`); + let clean = true; + const fail: Fail = (leg, line) => { + clean = false; + failLeg(leg, line); + }; + + const coordinatorKinds = await sectionKinds(page, "Coordinators"); + if (JSON.stringify(coordinatorKinds) !== JSON.stringify([fixture.coordinator.kind])) { + fail("roster", `the rail's Coordinators section lists the kinds ${JSON.stringify(coordinatorKinds)} (want ["${fixture.coordinator.kind}"])`); } - const mailboxes: string[] = []; - for (const kind of mailboxKinds) { + const conversations: string[] = []; + for (const kind of coordinatorKinds) { await open(page, kind); - const leaf = rail(page).locator(`ul[data-leaf="${kind}"]`); - await readUntil(() => leaf.locator("[data-session-id]").count(), (n) => n > 0, 10_000); - mailboxes.push(...(await leaf.locator("[data-session-id]").evaluateAll((bs) => bs.map((b) => b.getAttribute("data-session-id") ?? "")))); + const section = rail(page).getByRole("list", { name: "Coordinators" }); + await readUntil(() => section.locator("[data-session-id]").count(), (n) => n > 0, 10_000); + conversations.push(...(await section.locator("[data-session-id]").evaluateAll((bs) => bs.map((b) => b.textContent ?? "")))); } - if (JSON.stringify(mailboxes) !== JSON.stringify([MAILBOX])) { - fail("roster", `the rail lists the mailboxes ${JSON.stringify(mailboxes)} (want ["${MAILBOX}"])`); + if (JSON.stringify(conversations) !== JSON.stringify([COORDINATOR])) { + fail("roster", `the rail's Coordinators section opens on the conversations ${JSON.stringify(conversations)} (want ["${COORDINATOR}"])`); } - const seatKinds = await sectionKinds(page, "Seats"); - if (JSON.stringify(seatKinds) !== JSON.stringify([fixture.seatKind])) { - fail("roster", `the rail's Seats section lists the kinds ${JSON.stringify(seatKinds)} (want ["${fixture.seatKind}"])`); - } - // Every seat kind opened, then the seats read once: the section lists them all. - for (const kind of seatKinds) await open(page, kind); - const listed = rail(page).getByRole("list", { name: "Seats" }).locator("button[data-instance-id]"); - await readUntil(() => listed.count(), (n) => n > 0, 10_000); - const seats = await listed.evaluateAll((bs) => bs.map((b) => b.getAttribute("data-instance-id") ?? "")); - const want = fixture.specialists.map((s) => s.id); - if (JSON.stringify([...seats].sort()) !== JSON.stringify([...want].sort())) { - fail("roster", `the rail lists the seats ${JSON.stringify(seats)} (want ${JSON.stringify(want)})`); + const workerKinds = await sectionKinds(page, "Workers"); + if (JSON.stringify(workerKinds) !== JSON.stringify([fixture.workerKind])) { + fail("roster", `the rail's Workers section lists the kinds ${JSON.stringify(workerKinds)} (want ["${fixture.workerKind}"])`); } - // Each specialist, opened: its kind and its description, as drawn under its row. - const described: string[] = []; - for (const specialist of fixture.specialists) { - if (!seats.includes(specialist.id)) continue; - await open(page, specialist.id); - const detail = rail(page).locator(`[data-leaf-detail="${specialist.id}"]`); - await detail.waitFor({ timeout: 10_000 }); - const kind = (await detail.locator("[data-seat-kind]").textContent()) ?? ""; - const description = (await detail.getByTestId("seat-description").count()) === 1 - ? ((await detail.getByTestId("seat-description").textContent()) ?? "") - : "(none drawn)"; - if (kind !== fixture.seatKind) fail("roster", `${specialist.id} shows the kind "${kind}" (want "${fixture.seatKind}")`); - if (description !== specialist.description) { - fail("roster", `${specialist.id} shows the description ${JSON.stringify(description)} (want ${JSON.stringify(specialist.description)})`); - } else { - described.push(`${specialist.id}: ${JSON.stringify(description)}`); + const rows = await readUntil(() => rosterRows(page), (rs) => rs.length > 0, 10_000); + const want = [ + { id: COORDINATOR, flow: fixture.coordinator.kind, description: fixture.coordinator.description }, + ...fixture.specialists.map((s) => ({ id: s.id, flow: fixture.workerKind, description: s.description })), + ]; + const ids = rows.map((r) => r.id); + if (JSON.stringify([...ids].sort()) !== JSON.stringify(want.map((w) => w.id).sort())) { + fail("roster", `the roster panel lists ${JSON.stringify(ids)} (want ${JSON.stringify(want.map((w) => w.id))})`); + } + for (const w of want) { + const drawn = rows.find((r) => r.id === w.id); + if (drawn === undefined) continue; + if (drawn.flow !== w.flow) fail("roster", `${w.id} shows the flow "${drawn.flow}" (want "${w.flow}")`); + if (drawn.description !== w.description) { + fail("roster", `${w.id} shows the description ${JSON.stringify(drawn.description)} (want ${JSON.stringify(w.description)})`); } } + const hireButtons = await page.getByRole("button", { name: "Hire another" }).count(); if (hireButtons > 0) fail("roster", `the page draws ${hireButtons} "Hire another" button(s) (want none)`); - if (described.length === fixture.specialists.length && hireButtons === 0 && mailboxes.length === 1) { - evidence.push(`roster: the rail lists ${JSON.stringify(mailboxes)} under "${mailboxKinds.join(",")}" and four "${fixture.seatKind}" seats, ${described.join("; ")}; no "Hire another"`); - } -} - -/** One answer line under the seat, after the person's line carrying `mark`. */ -function oneLineBySeat(drawn: Drawn, mark: string): { line?: { label: string; text: string }; problem?: string } { - const answers = answersTo(drawn, mark); - if (answers === undefined) return { problem: `after the reload, no line of the person's carries ${mark}` }; - if (answers.length !== 1 || answers[0]!.label !== SEAT) { - return { problem: `after the reload, the lines answering ${mark} are ${JSON.stringify(answers.map((l) => `${l.label}: ${l.text}`))} (want one, by ${SEAT})` }; + if (clean) { + evidence.push( + `roster: the rail lists "${coordinatorKinds.join(",")}" opening on ${JSON.stringify(conversations)} and "${workerKinds.join(",")}"; ` + + `the roster panel lists ${rows.map((r) => `${r.id} (${r.flow}): ${JSON.stringify(r.description)}`).join("; ")}; no "Hire another"`, + ); } - return { line: answers[0] }; } // --------------------------------------------------------------------------- @@ -275,82 +263,50 @@ await runGoal(async () => { // ---- roster ------------------------------------------------------------ await gradeRoster(page, origin, fail, evidence); - // Every other leg posts to the mailbox the roster names. Without that + // The answer leg posts to the coordinator the roster names. Without that // roster there is nothing to post to, so the verdict is the roster's. if (failures.length > 0) return { failures, evidence: evidence.join("; ") }; - // ---- answer and file: two posts, one reload ----------------------------- + // ---- answer: one post, one reload ---------------------------------------- const answerToken = `answer-token-a${run}`; - const fileToken = `case-token-f${run}`; - await openMailbox(page, origin); + await openHelpDesk(page, origin); await post(page, `[route:${SEAT}] ${fixture.answer.marker} ${answerToken} where is my refund?`); await waitAnswered(page, origin, answerToken, 15_000); - await post(page, `[route:${SEAT}] ${fixture.file.marker} ${fileToken} the charger caught fire and the desk smells of smoke`); - await waitAnswered(page, origin, fileToken, 15_000); - // The row is written by the mailbox's own request, a moment after the - // dispatch. Let it land before the reload; nothing here is graded. - await readUntil( - async () => (await page.request.get(`${origin}/api/flows/sessions/${MAILBOX}/resources/${MAILBOX}.${fixture.mailbox.board}`)).text(), - (body) => body.includes(fileToken), - 10_000, - ); - - const drawn = await drawnAfterReload(page, origin, [answerToken, fileToken]); - const answered = oneLineBySeat(drawn, answerToken); - if (answered.problem !== undefined) fail("answer", answered.problem); - else if (!answered.line!.text.includes(fixture.answer.replyMarker)) { - fail("answer", `${SEAT}'s line ${JSON.stringify(answered.line!.text)} does not carry ${fixture.answer.replyMarker}`); - } else if (answered.line!.text.includes(answerToken)) { - fail("answer", `${SEAT}'s line hands the post back: ${JSON.stringify(answered.line!.text)}`); + const drawn = await drawnAfterReload(page, origin, [answerToken]); + const answers = answersTo(drawn, answerToken); + if (answers === undefined) { + fail("answer", `after the reload, no line of the person's carries ${answerToken}`); + } else if (answers.length !== 1 || answers[0]!.label !== SEAT) { + fail("answer", `after the reload, the lines answering ${answerToken} are ${JSON.stringify(answers.map((l) => `${l.label}: ${l.text}`))} (want one, by ${SEAT})`); + } else if (!answers[0]!.text.includes(fixture.answer.replyMarker)) { + fail("answer", `${SEAT}'s line ${JSON.stringify(answers[0]!.text)} does not carry ${fixture.answer.replyMarker}`); + } else if (answers[0]!.text.includes(answerToken)) { + fail("answer", `${SEAT}'s line hands the post back: ${JSON.stringify(answers[0]!.text)}`); } else { - evidence.push(`answer: after a reload, ${MAILBOX} shows one line answering ${answerToken}, by ${SEAT}: ${JSON.stringify(answered.line!.text)}`); + evidence.push(`answer: after a reload, ${COORDINATOR}'s conversation shows one line answering ${answerToken}, by ${SEAT}: ${JSON.stringify(answers[0]!.text)}`); } - const filed = oneLineBySeat(drawn, fileToken); - if (filed.problem !== undefined) fail("file:line", filed.problem); - else if (!filed.line!.text.includes(fixture.file.replyMarker)) { - fail("file:line", `${SEAT}'s line ${JSON.stringify(filed.line!.text)} does not say it filed (${fixture.file.replyMarker})`); - } else { - evidence.push(`file: ${SEAT}'s line reads ${JSON.stringify(filed.line!.text)}`); - } - const rows = await readUntil(() => boardRows(page), (rs) => rs.some((r) => r.includes(fileToken)), 10_000); - const mine = rows.filter((r) => r.includes(fileToken)); - if (mine.length !== 1) { - fail("file:row", `after the reload, the team panel's ${fixture.mailbox.board} board holds ${mine.length} rows carrying ${fileToken} (want 1); it shows ${rows.length} rows`); - } else { - evidence.push(`file: after a reload, the team panel's ${fixture.mailbox.board} board shows ${JSON.stringify(mine[0])}`); - } - - // ---- live: real models file what needs a person, and only that --------- + // ---- live: real models and the app's own route, one answer per post ----- if (LIVE) { if ((process.env.AI_GATEWAY_API_KEY ?? "") === "") throw new Error("GOAL_LIVE=1 needs AI_GATEWAY_API_KEY"); server.stop(); live = await startKitchenSink(fixture.port + 1, { KITCHEN_SINK_TEST_MODE: "" }); - const posts = [ - ...fixture.live.needsAPerson.map((text) => ({ text, files: true })), - ...fixture.live.answerable.map((text) => ({ text, files: false })), - ]; - await openMailbox(page, live.origin); - let before = (await boardRows(page)).length; - for (const [i, { text, files }] of posts.entries()) { + const specialists = fixture.specialists.map((s) => s.id); + for (const [i, text] of fixture.live.posts.entries()) { const mark = `(ref live-token-${i}${run})`; - await openMailbox(page, live.origin); + await openHelpDesk(page, live.origin); await post(page, `${text} ${mark}`); await waitAnswered(page, live.origin, mark, 120_000); - // Let a filing's row land before the reload; not graded. + // Let a second, wrong answer land before the reload, if one is coming; not graded. await page.waitForTimeout(3_000); - const liveDrawn = await drawnAfterReload(page, live.origin, [mark]); - const answers = answersTo(liveDrawn, mark) ?? []; - const after = (await boardRows(page)).length; - const who = answers.map((l) => l.label).join(", ") || "nobody"; - if (answers.length !== 1) fail("live", `post ${i + 1} has ${answers.length} answer line(s) (want 1): ${JSON.stringify(answers)}`); - if (after - before !== (files ? 1 : 0)) { - fail("live", `post ${i + 1} (${files ? "needs a person" : "answerable"}) added ${after - before} row(s) to ${fixture.mailbox.board} (want ${files ? 1 : 0}); answered by ${who}: ${JSON.stringify(answers[0]?.text ?? "")}`); + const liveAnswers = answersTo(await drawnAfterReload(page, live.origin, [mark]), mark) ?? []; + const who = liveAnswers.map((l) => l.label).join(", ") || "nobody"; + if (liveAnswers.length !== 1 || !specialists.includes(liveAnswers[0]!.label)) { + fail("live", `post ${i + 1} has ${liveAnswers.length} answer line(s) by ${who} (want one, by a specialist): ${JSON.stringify(liveAnswers)}`); } else { - evidence.push(`live ${i + 1}: ${files ? "filed" : "not filed"}, answered by ${who}: ${JSON.stringify(answers[0]?.text ?? "")}`); + evidence.push(`live ${i + 1}: answered by ${who}: ${JSON.stringify(liveAnswers[0]!.text)}`); } - before = after; } } } finally { @@ -359,16 +315,7 @@ await runGoal(async () => { live?.stop(); } - // The boot still warns that escalations is unattended, as the only such board. - const unattended = [...(server?.log() ?? "").matchAll(/mailbox "([^"]+)" holds board "([^"]+)"/g)].map((m) => `${m[1]}.${m[2]}`); - const unique = [...new Set(unattended)]; - if (JSON.stringify(unique) !== JSON.stringify([`${MAILBOX}.${fixture.mailbox.board}`])) { - fail("warning", `the boot warns these boards are unattended: ${JSON.stringify(unique)} (want only ${MAILBOX}.${fixture.mailbox.board})`); - } else { - evidence.push(`the boot warns that ${MAILBOX}.${fixture.mailbox.board} is unattended, and no other board`); - } - - // A control must redden each assertion it names, and only those. + // A control must redden each leg it names, and only those. if (CONTROL !== "") { const want = EXPECTED[CONTROL]!; const legs = new Set(failures.map((f) => /^\[([^\]]+)\]/.exec(f)?.[1] ?? "")); diff --git a/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/fixtures/input.json b/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/fixtures/input.json index f03ddd43c0..1963b009f8 100644 --- a/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/fixtures/input.json +++ b/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/fixtures/input.json @@ -1,5 +1,5 @@ { "port": 3037, - "mailbox": { "kind": "mailbox", "id": "support.help", "label": "devuser" }, + "coordinator": { "kind": "coordinator", "id": "support.help", "label": "devuser" }, "seat": { "kind": "agent", "id": "support.devices", "marker": "[scenario:talk-to-seat]", "replyMarker": "[reply:talk-to-seat]" } } diff --git a/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/goal.md b/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/goal.md index 2589a4ca69..9dfbe2a1c8 100644 --- a/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/goal.md +++ b/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/goal.md @@ -1,30 +1,32 @@ # kitchen-sink-talk › it keeps both sides across a reload -**Issue:** FIX-1585 (V6 and V7 of the spec's PLAN; the otto leg is the epic FIX-1592's leg a for `support.otto`); re-pointed by FIX-1611 +**Issue:** FIX-1585 (V6 and V7 of the spec's PLAN; the otto leg is the epic FIX-1592's leg a for `support.otto`); re-pointed by FIX-1611; converted by FIX-1792 P1 -**Outcome:** From the kitchen-sink page, a person talks to a specialist and posts to the support mailbox, and after a reload both conversations are still there, showing who said what. The line posted to `support.help` reads as `devuser`. The message sent to `support.devices` shows as the person's turn, with the seat's reply under it. +**Outcome:** From the kitchen-sink page, a person talks to a specialist and posts to the support desk's coordinator, and after a reload both conversations are still there, showing who said what. The line posted to `support.help` reads as `devuser`. The message sent to `support.devices` shows as the person's turn, with the specialist's reply under it. -**Re-pointed (FIX-1611).** The folder name is kept so its verdict history stays in one place. Until FIX-1611 this read `support.desk`, `support.otto`, and a third leg on `support.wren`, a seat whose kind took no messages. The legs are renamed for what they check: **desk** is now **mailbox**, **otto** is now **seat**. The wren leg has no seat to run on: every seat in this roster is an `agent`, and the rail lists only the kinds the shell names, so no seat of another kind reaches the page, whether hired or injected into the flow list. It moved to the panel itself, `apps/kitchen-sink/test/picked-session-panel.test.tsx` (a seat of a kind the shell has no ask for shows no composer and its reason); that case's red state is in FIX-1611's PR. +**Re-pointed (FIX-1611).** The folder name is kept so its verdict history stays in one place. Until FIX-1611 this read `support.desk`, `support.otto`, and a third leg on `support.wren`, a seat whose kind took no messages. The legs were renamed for what they check: **desk** became **mailbox**, **otto** became **seat**. The wren leg has no seat to run on: every seat in this roster is an `agent`, and the rail lists only the kinds the shell names, so no seat of another kind reaches the page, whether hired or injected into the flow list. It moved to the panel itself, `apps/kitchen-sink/test/picked-session-panel.test.tsx` (a seat of a kind the shell has no ask for shows no composer and its reason); that case's red state is in FIX-1611's PR. -**Input:** `fixtures/input.json`: the mailbox and its expected label, the specialist and the scenario marker its scripted reply keys on, and the port. Held-out: the posted line and the message carry a fresh token each run, and are graded by that token, so a line another run or test left in the shared mailbox can never be the one that passes. `GOAL_SEAT` points the seat leg at another specialist; a correct implementation still passes. The roster has one mailbox, so the mailbox leg has no override. +**Converted (FIX-1792 P1).** `support.help` is a coordinator worker file, no longer a mailbox, so the **mailbox** leg is now **coordinator**: the person's post is a `user` message in their conversation with `support.help`, drawn under `devuser`. Since FIX-1788 the rail has no row per specialist, so the seat leg starts its conversation with "Talk" on `support.devices` in the roster panel and reopens it from the `agent` flow's one copy in the rail; which session "Talk" opened is read from the server by the session's `workerId`, as an index. Both of the person's lines are now `user` messages, so each control names the flow it strips them from: `no-post-item` becomes `drop-coordinator-post`, and `drop-user-message` keeps its name and strips the `agent` flow's only. -**Signal:** one real browser against kitchen-sink's **production build** (built by the run, never assumed), served by `next start` on the scripted model and the in-memory store. Everything graded is read **after a reload**, so only what the server kept can pass. +**Input:** `fixtures/input.json`: the coordinator and its expected label, the specialist and the scenario marker its scripted reply keys on, and the port. Held-out: the posted line and the message carry a fresh token each run, and are graded by that token, so a line another run or test left behind can never be the one that passes. `GOAL_SEAT` points the seat leg at another specialist; a correct implementation still passes. The roster has one coordinator, so the coordinator leg has no override. -- **mailbox** (FIX-1585 V6): post a unique line from the mailbox's panel; reload; the mailbox's transcript holds exactly one copy, labelled `devuser`. -- **seat** (FIX-1585 V7): "New conversation" on the seat's row, send a unique message; reload and reopen that conversation; a `user` message holds the text and an `assistant` message carrying the scripted reply marker sits below it. +**Signal:** one real browser against kitchen-sink's **production build** (built by the run, never assumed), served by `next start` on the scripted model and the in-memory store, keyless. Everything graded is read **after a reload**, so only what the server kept can pass. -**Anti-game:** a hollow pass would grade the page before the reload (the composer's own state or a streamed copy would pass), or grade any line in the shared mailbox rather than this run's token. The check reads after the reload only, grades by the run's token, runs the seat leg on the `agent` kind, and checks the message's **role**, not only its text. A control that leaves its leg green, or reddens another leg, fails the run. +- **coordinator** (FIX-1585 V6): post a unique line from the coordinator's panel; reload; the conversation with `support.help` holds exactly one copy, labelled `devuser`. +- **seat** (FIX-1585 V7): "Talk" on the specialist in the roster, send a unique message; reload and reopen that conversation from the rail; a `user` message holds the text and an `assistant` message carrying the scripted reply marker sits below it. + +**Anti-game:** a hollow pass would grade the page before the reload (the composer's own state or a streamed copy would pass), or grade any line rather than this run's token. The check reads after the reload only, grades by the run's token, runs the seat leg on the `agent` flow, and checks the message's **role**, not only its text. A control that leaves its leg green, or reddens another leg, fails the run. **Model:** n/a. kitchen-sink's scripted model answers the seat (epic FIX-1592 D3). The goal is what is kept and who said it, not what the reply says. Keyless. **Run:** `PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/run.mts` -**Controls:** on the same command. Each strips one kind of kept item from everything the page is served (every session read, every action stream and the live session stream, before and after the reload), which is what the page sees when the server keeps none. +**Controls:** on the same command. Each strips the person's `user` messages of one flow from everything the page is served (every session read, every action stream and the live session stream, before and after the reload), which is what the page sees when the server keeps none on that side. A session read's flow is learned from the session's own state. -- `GOAL_CONTROL=no-post-item`: no `mailbox-post` component items. Must FAIL at **mailbox** only. -- `GOAL_CONTROL=drop-user-message`: no `user` message items. Must FAIL at **seat** only. +- `GOAL_CONTROL=drop-coordinator-post`: the `coordinator` flow's. Must FAIL at **coordinator** only. +- `GOAL_CONTROL=drop-user-message`: the `agent` flow's. Must FAIL at **seat** only. -The controls act at the page's `fetch`, not in the server, so the product carries no switch for them. Not with Playwright's routing: the live session stream never ends, and a routed response is read whole before the page gets any of it, so each stream is filtered frame by frame as it arrives. The server half is proved by hand mutation, recorded below: `run` without its `userMessage`, rebuilt, fails the seat leg (then named otto). +The controls act at the page's `fetch`, not in the server, so the product carries no switch for them. Not with Playwright's routing: the live session stream never ends, and a routed response is read whole before the page gets any of it, so each stream is filtered frame by frame as it arrives. The server half is proved by hand mutation, recorded below (FIX-1611's): `run` without its `userMessage`, rebuilt, fails the seat leg (then named otto). ## Verdict log | Date | Commit | Model | Verdict | Notes | @@ -43,3 +45,7 @@ The controls act at the page's `fetch`, not in the server, so the product carrie | 2026-09-28 | `fix/fix-1623` (FIX-1625) on 9a55b79e4, uncommitted | scripted | FAIL (control) | **Filtered at the page's `fetch`**, the live session stream included. `GOAL_CONTROL=no-post-item`: failed at **mailbox** only: after the reload `support.help`'s transcript holds 0 copies of the posted line, and 0 lines. **seat** green. | | 2026-09-28 | `fix/fix-1623` (FIX-1625) on 9a55b79e4, uncommitted | scripted | FAIL (control) | `GOAL_CONTROL=drop-user-message`: failed at **seat** only: after the reload `support.devices`' conversation does not hold the person's message as their turn (roles on screen: `assistant`). **mailbox** green. | | 2026-09-28 | `fix/fix-1623` (FIX-1625) on 9a55b79e4, uncommitted | scripted | **PASS** | Fresh production build. After a reload `support.help` shows `goal line …` labelled `devuser`, once; `support.devices`' new conversation shows the message as the user's turn with the `[reply:talk-to-seat]` reply under it. | +| 2026-10-09 | `main` at 57d2c10e9 | scripted | FAIL | **Taken first, on clean `main`, before the conversion.** The previous check fails at setup: its seat leg presses "New conversation" on a rail row named for the specialist, and since FIX-1788 P4 the rail lists one `agent` copy and no row per worker. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Converted.** Fresh production build, keyless. After a reload the conversation with `support.help` shows `goal line …` labelled `devuser`, once; `support.devices`' conversation (`wks_…`, opened by "Talk" and reopened from the rail) shows the message as the user's turn with the `[reply:talk-to-seat]` reply under it. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=drop-coordinator-post`: **coordinator** only — after the reload the conversation holds 0 copies of the posted line, and 1 line in all. **seat** green. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=drop-user-message`: **seat** only — the conversation does not hold the person's message as their turn (roles on screen: `assistant`). **coordinator** green. | diff --git a/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/run.mts b/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/run.mts index bc493d14b8..7aeaa9cc19 100644 --- a/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/run.mts +++ b/goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/run.mts @@ -1,19 +1,20 @@ /** * Goal check: from the kitchen-sink page, a person talks to a specialist and - * posts to the support mailbox, and after a reload both conversations are - * still there, showing who said what. + * posts to the support desk's coordinator, and after a reload both + * conversations are still there, showing who said what. * * Real path, scripted model, out of CI. See goal.md for the contract. * * Two legs, in one real browser against the app's PRODUCTION build (built * here, never assumed), on its scripted model: * - * mailbox post a unique line to `support.help` from its panel; reload; the - * line is in the mailbox's transcript, labelled with the app's one - * user. - * seat start a conversation on `support.devices` from its row, send a - * unique message; reload; the message is there as the person's - * turn and the scripted reply is under it. + * coordinator post a unique line to `support.help` from its panel; reload; + * the line is in the person's conversation with the + * coordinator, labelled with the app's one user. + * seat "Talk" on `support.devices` in the roster, send a unique + * message; reload and reopen that conversation from the rail; + * the message is there as the person's turn and the scripted + * reply is under it. * * The third leg this check once had, a seat whose kind takes no messages, has * no seat to run on: every seat in this roster is an `agent`, and the rail @@ -24,8 +25,8 @@ * pass. The page draws no optimistic copy of either side. * * Run: PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/keeps-both-sides-across-a-reload/run.mts - * Controls: GOAL_CONTROL=no-post-item (must FAIL at the mailbox leg only) - * GOAL_CONTROL=drop-user-message (must FAIL at the seat leg only) + * Controls: GOAL_CONTROL=drop-coordinator-post (must FAIL at the coordinator leg only) + * GOAL_CONTROL=drop-user-message (must FAIL at the seat leg only) * Held-out: GOAL_SEAT= */ import { randomUUID } from "node:crypto"; @@ -34,49 +35,63 @@ import { loadFixture, runGoal } from "../../lib/index.mts"; import { buildKitchenSink, conversation, - newConversation, - open, + COORDINATOR_FLOW, openShell as openShellAt, + openWorkerCopy, panel, - rail, readUntil, - row, + showCoordinator, startKitchenSink, + talkTo, + workerSessions, + WORKER_FLOW, type KitchenSinkServer, } from "../../lib/kitchen-sink.mts"; import { launchChromium } from "../../lib/playwright.mts"; interface Fixture { port: number; - mailbox: { kind: string; id: string; label: string }; + coordinator: { kind: string; id: string; label: string }; seat: { kind: string; id: string; marker: string; replyMarker: string }; } const fixture = loadFixture(import.meta.url); const ORIGIN = `http://127.0.0.1:${fixture.port}`; -// The roster has one mailbox, so only the seat has a held-out override. -const MAILBOX = fixture.mailbox.id; +// The roster has one coordinator, so only the seat has a held-out override. +const COORDINATOR = fixture.coordinator.id; const SEAT = process.env.GOAL_SEAT ?? fixture.seat.id; const CONTROL = process.env.GOAL_CONTROL ?? ""; /** The one leg each control must redden, and only that one. */ const EXPECTED: Record = { - "no-post-item": "mailbox", + "drop-coordinator-post": "coordinator", "drop-user-message": "seat", }; if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); } +/** The flow whose sessions each control strips the person's messages from. */ +const STRIPPED_FLOW: Record = { + "drop-coordinator-post": COORDINATOR_FLOW, + "drop-user-message": WORKER_FLOW, +}; + // --------------------------------------------------------------------------- // The controls: what the page is served, with one kept side taken out // --------------------------------------------------------------------------- /** * Runs in the page before its own scripts, under a control. Every session read, - * every action stream and every live session stream reaches the page without - * the one kind of item the control names, before and after the reload alike: - * what the page sees when the server keeps none. + * every action stream and every live session stream of the control's flow + * reaches the page without the person's messages, before and after the reload + * alike: what the page sees when the server keeps none on that side. The + * person's post to the coordinator and their message to the seat are both + * `user` messages, so each control names the flow it strips them from. + * + * An action's flow is in its path. A session read or stream carries only the + * session's id, so its flow is learned from the session's state, which names + * it (read once per session, unfiltered). * * At the page's `fetch`, not with Playwright's routing: the live session * stream (`GET /api/flows/sessions/:id/stream`) never ends, and a routed @@ -87,21 +102,28 @@ if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { * tsx first, which adds helpers the page does not have. */ const STRIP_ITEMS = `(() => { - const control = ${JSON.stringify(CONTROL)}; - const stripped = (item) => - item != null && - (control === "no-post-item" - ? item.type === "component" && item.component === "mailbox-post" - : item.type === "message" && item.role === "user"); + const flow = ${JSON.stringify(STRIPPED_FLOW[CONTROL] ?? "")}; + const stripped = (item) => item != null && item.type === "message" && item.role === "user"; const original = window.fetch.bind(window); + const flows = new Map(); + const flowOf = (sessionId) => { + if (!flows.has(sessionId)) { + flows.set(sessionId, original("/api/flows/sessions/" + encodeURIComponent(sessionId) + "/state?limit=1") + .then((res) => res.json()) + .then((json) => json.flowKind) + .catch(() => undefined)); + } + return flows.get(sessionId); + }; window.fetch = async (input, init) => { const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, location.href); const method = String((init && init.method) || (input instanceof Request ? input.method : "GET")).toUpperCase(); - const flows = url.pathname.startsWith("/api/flows/"); - const read = method === "GET" && /\\/sessions\\/[^/]+\\/(state|stream)$/.test(url.pathname); - const action = method === "POST" && /\\/actions\\/[^/]+$/.test(url.pathname); + const read = method === "GET" ? /^\\/api\\/flows\\/sessions\\/([^/]+)\\/(state|stream)$/.exec(url.pathname) : null; + const action = method === "POST" ? /^\\/api\\/flows\\/([^/]+)\\/actions\\/[^/]+$/.exec(url.pathname) : null; + if (read === null && action === null) return original(input, init); + const kind = action !== null ? decodeURIComponent(action[1]) : await flowOf(decodeURIComponent(read[1])); const response = await original(input, init); - if (!flows || (!read && !action)) return response; + if (kind !== flow) return response; const type = response.headers.get("content-type") || ""; if (type.includes("application/json")) { const json = await response.json(); @@ -151,14 +173,14 @@ async function applyControl(page: Page): Promise { const openShell = (page: Page) => openShellAt(page, ORIGIN); -/** The mailbox's transcript as drawn: each line's label and body. */ +/** The coordinator conversation's transcript as drawn: each line's label and body. */ const transcript = (page: Page) => panel(page) - .locator('[data-testid="mailbox-line"]') + .locator('[data-testid="coordinator-line"]') .evaluateAll((lines) => lines.map((line) => ({ - label: line.querySelector('[data-testid="mailbox-line-label"]')?.textContent ?? "", - body: line.querySelector('[data-testid="mailbox-line-body"]')?.textContent ?? "", + label: line.querySelector('[data-testid="coordinator-line-label"]')?.textContent ?? "", + body: line.querySelector('[data-testid="coordinator-line-body"]')?.textContent ?? "", })), ); @@ -177,52 +199,47 @@ await runGoal(async () => { const browser = await launchChromium(); let server: KitchenSinkServer | undefined; try { - server = await startKitchenSink(fixture.port); + // Keyless: the scripted model answers, and no key is there to fall back on. + server = await startKitchenSink(fixture.port, { AI_GATEWAY_API_KEY: "" }); const page = await browser.newPage({ viewport: { width: 1440, height: 900 } }); await applyControl(page); - // ---- mailbox: post, reload, read ------------------------------------- + // ---- coordinator: post, reload, read --------------------------------- await openShell(page); - await open(page, fixture.mailbox.kind); - await row(page, MAILBOX).click(); - await panel(page).getByLabel("Post to this mailbox").fill(line); + await showCoordinator(page, COORDINATOR); + await panel(page).getByLabel("Post to this coordinator").fill(line); await panel(page).getByRole("button", { name: "Send" }).click(); // Let the post settle before the reload; nothing here is graded. - await readUntil(() => panel(page).getByLabel("Post to this mailbox").inputValue(), (v) => v === "", 10_000); + await readUntil(() => panel(page).getByLabel("Post to this coordinator").inputValue(), (v) => v === "", 10_000); await page.reload(); await openShell(page); - await open(page, fixture.mailbox.kind); - await row(page, MAILBOX).click(); + await showCoordinator(page, COORDINATOR); const lines = await readUntil(() => transcript(page), (ls) => ls.some((l) => l.body === line)); const found = lines.filter((l) => l.body === line); if (found.length !== 1) { - fail("mailbox", `after the reload, ${MAILBOX}'s transcript holds ${found.length} copies of the posted line "${line}" (want 1); it shows ${lines.length} lines`); - } else if (found[0]!.label !== fixture.mailbox.label) { - fail("mailbox", `the posted line reads as "${found[0]!.label}", not "${fixture.mailbox.label}"`); + fail("coordinator", `after the reload, ${COORDINATOR}'s conversation holds ${found.length} copies of the posted line "${line}" (want 1); it shows ${lines.length} lines`); + } else if (found[0]!.label !== fixture.coordinator.label) { + fail("coordinator", `the posted line reads as "${found[0]!.label}", not "${fixture.coordinator.label}"`); } else { - evidence.push(`mailbox: after a reload, ${MAILBOX} shows "${line}" labelled ${found[0]!.label}, once`); + evidence.push(`coordinator: after a reload, ${COORDINATOR}'s conversation shows "${line}" labelled ${found[0]!.label}, once`); } - // ---- seat: a new conversation, a message, reload, read --------------- - await open(page, fixture.seat.kind); - await open(page, SEAT); - await newConversation(page, SEAT); - const sessionId = await rail(page) - .locator(`ul[data-leaf="${SEAT}"] [aria-current="true"]`) - .getAttribute("data-session-id"); + // ---- seat: Talk, a message, reload, reopen from the rail, read --------- + await talkTo(page, SEAT); await panel(page).getByLabel("Message this seat").fill(message); await panel(page).getByRole("button", { name: "Send" }).click(); await readUntil(() => conversation(page), (ms) => ms.some((m) => m.text.includes(fixture.seat.replyMarker))); + // Which session "Talk" opened, from the server's index by worker. Not graded. + const [talked, ...more] = await workerSessions(page, ORIGIN, SEAT); - if (sessionId === null) { - fail("seat", `"New conversation" on ${SEAT} opened no conversation in the rail`); + if (talked === undefined || more.length > 0) { + fail("seat", `"Talk" on ${SEAT} left the person ${more.length + (talked === undefined ? 0 : 1)} conversations with it (want 1)`); } else { await page.reload(); await openShell(page); - await open(page, fixture.seat.kind); - await open(page, SEAT); - await rail(page).locator(`[data-session-id="${sessionId}"]`).click(); + const leaf = await openWorkerCopy(page); + await leaf.locator(`[data-session-id="${talked.id}"]`).click(); const messages = await readUntil( () => conversation(page), (ms) => ms.some((m) => m.text.includes(message)) && ms.some((m) => m.text.includes(fixture.seat.replyMarker)), @@ -238,10 +255,9 @@ await runGoal(async () => { fail("seat", `the reply sits above the message it answers`); } if (asked !== -1 && answered > asked) { - evidence.push(`seat: after a reload, ${SEAT}'s conversation ${sessionId} shows the message as the user's turn and the scripted reply under it`); + evidence.push(`seat: after a reload, ${SEAT}'s conversation ${talked.id}, reopened from the rail, shows the message as the user's turn and the scripted reply under it`); } } - } finally { await browser.close(); server?.stop(); diff --git a/goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/fixtures/input.json b/goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/fixtures/input.json deleted file mode 100644 index 4d45d2839f..0000000000 --- a/goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/fixtures/input.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "port": 3042, - "mailbox": { "kind": "mailbox", "id": "support.help", "board": "escalations" }, - "seat": "support.devices", - "specialists": ["support.devices", "support.accounts", "support.fsd", "support.general"], - "marker": "[scenario:needs-a-person]", - "status": "pending", - "rowWithinMs": 10000, - "maxBoardReads": 2, - "idleMs": 10000 -} diff --git a/goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/goal.md b/goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/goal.md deleted file mode 100644 index 2116ba3a87..0000000000 --- a/goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/goal.md +++ /dev/null @@ -1,42 +0,0 @@ -# kitchen-sink-talk › lists a filed case without a reload - -**Issue:** FIX-1622 (VG of the spec's PLAN; the epic FIX-1592). FIX-1601's closure (P2e) re-runs it. - -**Outcome:** A person who asks `support.help` for a real person sees the case appear in the escalations panel as a row with its status, in a list, with no reload, in that tab and in any other tab open on the demo. - -**Input:** `fixtures/input.json`: the mailbox and its board (`support.help`, `escalations`), the specialist a case is routed to (`support.devices`), the scenario marker the scripted specialist files on (`[scenario:needs-a-person]`), the status every filed row shows (`pending`, since nothing works the board), how long a row may take (10 s), how many board reads a tab may make meanwhile (2), and the idle window (10 s). Held-out: each run files two cases, each with a fresh `case-token-…`, and only those tokens are graded, so a row another run left on the shared board can never be the one that passes. `GOAL_SEAT` routes both cases to another specialist. - -**Signal:** two tabs on kitchen-sink's **production build** (built by the run, never assumed), served by `next start` on the scripted model, the in-memory store and **no model key**. Tab 2 is opened first and never picks anything; tab 1 opens `support.help` and posts both cases, the second once the first has settled. Everything graded is read off the two pages as drawn; each tab is reloaded once, for the last leg only. - -- **row**: within 10 s of Send, tab 1's escalations panel holds a row carrying the case's token. The second case's row shows above the first's (newest first). -- **other-tab**: the same, in tab 2. -- **status**: every time a case's row is drawn, in either tab, open or after the reload, its status word reads `pending`, and the board draws no status column (`[data-column]`). A row drawn nowhere fails it: there is no status to read. -- **once**: no reading while the tabs are open shows a case's row twice, and after the final reload each case is exactly one row in each tab. -- **no-poll**: each tab reads the board (`GET …/sessions//resources/support.help.escalations`, through whichever session) at most twice between Send and the row, and not at all in a 10 s window after the first case has settled. - -**Anti-game:** the panel as drawn, never the ledger, the specialist's "filed" line, or a response. A reload or navigation in either tab before **once** voids the run (`void`). Tab 2 never picks the mailbox, so a list that re-reads when its own page sends cannot pass **other-tab**. The run reads the server only to know when a case has settled (the specialist's line is kept, and the board holds the row); that read is not graded. Only this run's tokens count. - -**Model:** n/a. kitchen-sink's scripted model routes each post by the specialist it names (`[route:…]`), and a specialist sent `[scenario:needs-a-person]` files the case onto `escalations` through the mailbox and says so. Keyless: the server runs with `AI_GATEWAY_API_KEY` empty. - -**Run:** `PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/run.mts` - -**Controls:** on the same command. - -- `GOAL_CONTROL=no-live`: the page is opened with `?goalControl=no-live`, which a test-mode build honours (`apps/kitchen-sink/lib/goal-control.ts`, `pageGoalControl`) by leaving the rail's panels and the team panel's boards without `live`. Nothing shows until the reload, where the list is right. Must FAIL at **row** and **other-tab**, and at nothing else. -- `GOAL_CONTROL=main`: the app before the list, run from a checkout of `main` with this directory copied in. Its board draws status columns, read once through the assistant's session. The app knows no control by that name; it tells the run which legs must fail. Must FAIL at **row**, **other-tab** and **status**, and at nothing else. -- `GOAL_CONTROL=no-filing`: the server swaps the specialist's `escalate` for a stand-in that says it filed and files nothing (`apps/kitchen-sink/lib/escalate-control.ts`). The line still says "filed", so it must FAIL at every leg but **no-poll**: **row**, **other-tab**, **status** and **once**. - -## Verdict log -| Date | Commit | Model | Verdict | Notes | -|------|--------|-------|---------|-------| -| 2026-09-28 | FIX-1622 branch at 91a5afdc4, this directory uncommitted | scripted | **PASS** | First run, taken before any control, so it proves nothing on its own; re-taken below after all three controls failed. Fresh production build, keyless. Case 1: tab 1 row +0.8s, tab 2 row +0.2s, one board read each. No board read in either tab in the 10s idle window. Case 2: tab 1 +0.7s, tab 2 +0.1s, above case 1, one board read each. After the reload each case is one row in each tab; every row drawn reads `pending`, in one list. | -| 2026-09-28 | FIX-1622 branch at 91a5afdc4, this directory uncommitted | scripted | FAIL (control) | `GOAL_CONTROL=no-live`. Failed at **row** and **other-tab** (both cases, both tabs: no row within 10s, no reload) and nothing else. After the reload the list showed each case once, `pending`, no columns. | -| 2026-09-28 | FIX-1622 branch at 91a5afdc4, this directory uncommitted | scripted | FAIL (control) | `GOAL_CONTROL=no-filing`. Failed at **row**, **other-tab**, **once** (0 rows per case per tab after the reload) and **status** (no row ever drawn), and nothing else. **no-poll** green: nothing changed, so nothing was read. | -| 2026-09-28 | `main` at cb00fc4f0, this directory copied in | scripted | FAIL (control) | `GOAL_CONTROL=main`. Failed at **row** and **other-tab** (no row in either tab, either case, before the reload) and **status** (after the reload each row sits in one of 7 status columns and draws no status word), and nothing else. **once** and **no-poll** green. | -| 2026-09-28 | FIX-1622 branch at 91a5afdc4, this directory uncommitted | scripted | **PASS** | Re-taken after the three controls. Case 1: tab 1 +0.7s, tab 2 +0.2s; case 2: tab 1 +0.7s, tab 2 +0.3s, above case 1; one board read per tab per case; none in the idle window. Each case once per tab after the reload, `pending`, one list. | -| 2026-09-28 | FIX-1622 branch at 91a5afdc4, this directory uncommitted | scripted | **PASS** | Held-out: `GOAL_SEAT=support.accounts`. Case 1: tab 1 +0.7s, tab 2 +0.2s; case 2: tab 1 +0.8s, tab 2 +0.3s, above case 1; one board read per tab per case; none idle. Each case once per tab after the reload, `pending`, one list. | -| 2026-09-28 | FIX-1622 branch at a55b0b454 (review round 1: a live list reads on mount and once more when its stream connects, with no clock margin; `baseUrl` prop) | scripted | **PASS** | Fresh production build, keyless. Case 1: tab 1 +0.7s, tab 2 +0.2s; case 2: tab 1 +0.8s, tab 2 +0.4s, above case 1. One board read per tab per case between Send and the row (the two mount reads land before Send); none in either tab in the 10s idle window. After the reload each case is one row in each tab; every row drawn reads `pending`, in one list. | -| 2026-09-28 | same | scripted | FAIL (control) | `GOAL_CONTROL=no-live`. Failed at **row** and **other-tab** (both cases, both tabs: no row within 10s, no reload) and nothing else; the run found no `[control]` problem. | -| 2026-09-28 | FIX-1622 branch at 8a1b6ac72 (review round 2: every change to the board reads it, the stream's opening scan included; no read on connecting) | scripted | **PASS** | Fresh production build, keyless. Case 1: tab 1 +0.7s, tab 2 +0.2s; case 2: tab 1 +0.9s, tab 2 +0.3s, above case 1. One board read per tab per case between Send and the row; none in either tab in the 10s idle window. After the reload each case is one row in each tab; every row drawn reads `pending`, in one list. | -| 2026-09-28 | same | scripted | FAIL (control) | `GOAL_CONTROL=no-live`. Failed at **row** and **other-tab** (both cases, both tabs: no row within 10s, no reload) and nothing else; the run found no `[control]` problem. | -| 2026-09-28 | FIX-1622 branch at 402d9741e (code as at 8a1b6ac72; 402d9741e changes only this file) | scripted | **PASS** | Re-taken after the `no-live` control above, on the same code, so the PASS follows its control. Fresh production build, keyless. Case 1: tab 1 +0.8s, tab 2 +0.2s; case 2: tab 1 +0.6s, tab 2 +0.2s, above case 1. One board read per tab per case between Send and the row; none in either tab in the 10s idle window. After the reload each case is one row in each tab; every row drawn reads `pending`, in one list. | diff --git a/goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/run.mts b/goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/run.mts deleted file mode 100644 index 1ff6f7f76f..0000000000 --- a/goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/run.mts +++ /dev/null @@ -1,398 +0,0 @@ -/** - * Goal check: a person who asks `support.help` for a real person sees the case - * appear in the escalations panel as a row with its status, in a list, with no - * reload, in that tab and in any other tab open on the demo. - * - * Real path, scripted model, out of CI. See goal.md for the contract. - * - * Two tabs on the app's PRODUCTION build (built here, never assumed), on its - * scripted model, keyless. Tab 2 is opened first and never picks anything. - * Tab 1 opens `support.help` and posts two cases that need a person, each with - * a fresh token, the second once the first has settled. Neither tab is - * reloaded until the last leg. Five legs: - * - * row within 10 s of Send, tab 1's escalations panel holds a row - * carrying the case's token; the second case's row shows above - * the first's. - * other-tab the same, in tab 2. - * status every time a case's row is drawn, in either tab, open or after - * the reload, it reads `pending`, and the board draws no status - * column. - * once no reading shows a case's row twice, and after the final - * reload each case is one row in each tab. - * no-poll at most two board reads per tab between Send and the row, and - * none in a 10 s window once the first case has settled. - * - * Everything graded is read off the pages. The server is read only to know - * when a case has settled, and nothing it says is graded. - * - * Run: PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/lists-a-filed-case-without-a-reload/run.mts - * Controls: GOAL_CONTROL=no-live (must FAIL at row and other-tab, and nothing else) - * GOAL_CONTROL=main (the app before the list, from a checkout of it: row, other-tab and status) - * GOAL_CONTROL=no-filing (every leg but no-poll) - * Held-out: GOAL_SEAT= - */ -import { randomUUID } from "node:crypto"; -import type { Page } from "playwright"; -import { loadFixture, runGoal } from "../../lib/index.mts"; -import { - buildKitchenSink, - open, - panel, - readUntil, - row, - startKitchenSink, - type KitchenSinkServer, -} from "../../lib/kitchen-sink.mts"; -import { launchChromium } from "../../lib/playwright.mts"; - -interface Fixture { - port: number; - mailbox: { kind: string; id: string; board: string }; - seat: string; - specialists: string[]; - /** What a post carries for the scripted specialist to file it. */ - marker: string; - /** The status word every filed row shows: nothing works the board. */ - status: string; - rowWithinMs: number; - maxBoardReads: number; - idleMs: number; -} - -const fixture = loadFixture(import.meta.url); -const SEAT = process.env.GOAL_SEAT ?? fixture.seat; -const CONTROL = process.env.GOAL_CONTROL ?? ""; -const MAILBOX = fixture.mailbox.id; -const BOARD_REF = `${MAILBOX}.${fixture.mailbox.board}`; - -/** The legs each control must redden, and only those. */ -const EXPECTED: Record = { - // The page's panels don't follow their session: no row until the reload, - // where it shows as a list with its status, once. - "no-live": ["row", "other-tab"], - // The app before the list, run from a checkout of it with this directory - // copied in: status columns, read once. The app knows no control by this - // name; it tells this run which legs must fail. - main: ["row", "other-tab", "status"], - // The specialist says it filed and files nothing. No row anywhere, so the - // "filed" line alone never passes; nothing changes, so nothing is read. - "no-filing": ["row", "other-tab", "status", "once"], -}; -if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { - throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); -} -if (!fixture.specialists.includes(SEAT)) { - throw new Error(`GOAL_SEAT "${SEAT}" is not one of the specialists: ${fixture.specialists.join(", ")}`); -} - -/** - * The page's query. `no-live` lives in the browser, and a client component - * cannot read `GOAL_CONTROL`, so the page is opened with it instead; the app - * honours it only in a test-mode build. - */ -const PAGE_QUERY = CONTROL === "no-live" ? "?goalControl=no-live" : ""; - -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); -const secs = (from: number, to: number | undefined) => (to === undefined ? "never" : `+${((to - from) / 1000).toFixed(1)}s`); - -// --------------------------------------------------------------------------- -// The pages, as a person uses them -// --------------------------------------------------------------------------- - -/** Load the page, with the control's query, and wait until it can be used. */ -async function openPage(page: Page, origin: string): Promise { - await page.goto(`${origin}/${PAGE_QUERY}`); - await page.locator('[data-testid="message-input"]:visible').waitFor({ state: "visible", timeout: 30_000 }); -} - -/** Open the mailbox's panel. */ -async function openMailbox(page: Page, origin: string): Promise { - await openPage(page, origin); - await open(page, fixture.mailbox.kind); - await row(page, MAILBOX).click(); - await panel(page).getByTestId("mailbox-transcript").waitFor({ timeout: 15_000 }); -} - -/** The escalations board in the team panel. */ -const boardPanel = (page: Page) => page.getByTestId(`board-${BOARD_REF}`); - -/** Wait until a tab's board has been read: drawn, and no longer loading. Not graded. */ -async function boardDrawn(page: Page): Promise { - await boardPanel(page).locator('[data-panel="board"]').first().waitFor({ timeout: 15_000 }); - await readUntil(() => boardPanel(page).locator('[data-state="loading"]').count(), (n) => n === 0, 10_000); -} - -interface DrawnRow { - text: string; - /** The row's status word, or `null` when the row draws none. */ - status: string | null; -} - -interface Reading { - at: number; - rows: DrawnRow[]; - /** How many status columns the board draws. */ - columns: number; -} - -/** The board as drawn: each row, in order, and how many status columns. */ -async function readBoard(page: Page): Promise { - const drawn = await boardPanel(page).evaluate((el) => ({ - rows: [...el.querySelectorAll("li[data-task-id]")].map((li) => ({ - text: li.textContent ?? "", - status: li.querySelector("[data-task-status]")?.textContent ?? null, - })), - columns: el.querySelectorAll("[data-column]").length, - })); - return { at: Date.now(), ...drawn }; -} - -/** When a tab read the board, through whichever session (`GET …/sessions//resources/`). */ -function boardReads(page: Page): number[] { - const at: number[] = []; - page.on("request", (request) => { - if (request.method() !== "GET") return; - const path = new URL(request.url()).pathname; - if (path.startsWith("/api/flows/sessions/") && path.endsWith(`/resources/${BOARD_REF}`)) at.push(Date.now()); - }); - return at; -} - -/** Post a line from the mailbox's panel once its composer is free. Returns when Send was pressed. */ -async function send(page: Page, line: string): Promise { - const box = panel(page).getByLabel("Post to this mailbox"); - await readUntil(async () => (await box.isEnabled()) && (await box.inputValue()) === "", (free) => free, 10_000); - await box.fill(line); - await panel(page).getByRole("button", { name: "Send" }).click(); - return Date.now(); -} - -type Line = { author?: string; body: string }; - -/** - * Wait until the case carrying `token` has settled: a seat's line follows the - * person's, and the board keeps its row. Read off the server, never graded; - * under a control that files nothing, it simply runs out. - */ -async function settled(page: Page, origin: string, token: string): Promise { - await readUntil( - async () => { - const res = await page.request.get(`${origin}/api/flows/sessions/${MAILBOX}/state?include_items=true&item_types=component&limit=1000`); - const items = ((await res.json()) as { items?: Array<{ component?: string; data?: Line }> }).items ?? []; - return items.filter((item) => item.component === "mailbox-post").map((item) => item.data!); - }, - (lines) => { - const at = lines.findIndex((l) => l.author === undefined && l.body.includes(token)); - return at !== -1 && lines.slice(at + 1).some((l) => l.author !== undefined); - }, - 20_000, - ); - await readUntil( - async () => (await page.request.get(`${origin}/api/flows/sessions/${MAILBOX}/resources/${BOARD_REF}`)).text(), - (body) => body.includes(token), - 10_000, - ); -} - -// --------------------------------------------------------------------------- - -interface Tab { - /** The leg this tab's rows are graded under. */ - leg: "row" | "other-tab"; - name: string; - page: Page; - reads: number[]; - /** Full page loads since setup. */ - loads: number; -} - -interface Case { - token: string; - line: string; - sentAt: number; - /** Per tab: when the row first showed, where it sat, and the most copies one reading held. */ - seen: Map; -} - -await runGoal(async () => { - const failures: string[] = []; - const evidence: string[] = []; - const fail = (leg: string, line: string) => failures.push(`[${leg}] ${line}`); - const run = randomUUID().replace(/-/g, "").slice(0, 10); - - // Every status problem once, however many readings showed it. - const statusProblems = new Set(); - let statusRowsRead = 0; - const checkStatus = (where: string, reading: Reading, token: string) => { - for (const drawn of reading.rows.filter((r) => r.text.includes(token))) { - statusRowsRead += 1; - if (drawn.status !== fixture.status) { - statusProblems.add(`${where}: the row carrying ${token} shows the status ${drawn.status === null ? "(none drawn)" : JSON.stringify(drawn.status)} (want "${fixture.status}")`); - } - } - if (reading.columns > 0 && reading.rows.some((r) => r.text.includes(token))) { - statusProblems.add(`${where}: the board draws ${reading.columns} status columns (want one list)`); - } - }; - - const cases: Case[] = [ - "the charger caught fire and the desk smells of smoke", - "the replacement charger is sparking too", - ].map((ask, i) => { - const token = `case-token-${"ab"[i]}${run}`; - return { token, line: `[route:${SEAT}] ${fixture.marker} ${token} ${ask}`, sentAt: 0, seen: new Map() }; - }); - - /** Fold one reading of a tab's board into what each sent case has seen. */ - const observe = (tab: Tab, reading: Reading) => { - for (const c of cases) { - if (c.sentAt === 0) continue; - const mine = reading.rows.filter((r) => r.text.includes(c.token)); - const seen = c.seen.get(tab) ?? { most: 0 }; - seen.most = Math.max(seen.most, mine.length); - if (mine.length > 0) { - checkStatus(`${tab.name}, open`, reading, c.token); - if (seen.rowAt === undefined) { - seen.rowAt = reading.at; - seen.order = reading.rows.map((r) => cases.find((k) => r.text.includes(k.token))?.token ?? ""); - } - } - c.seen.set(tab, seen); - } - }; - - buildKitchenSink(); - - const browser = await launchChromium(); - let server: KitchenSinkServer | undefined; - try { - // Keyless: the scripted model answers, and no key is there to fall back on. - server = await startKitchenSink(fixture.port, { AI_GATEWAY_API_KEY: "" }); - const origin = server.origin; - - // Tab 2 first, and it never picks anything. - const otherPage = await browser.newPage({ viewport: { width: 1440, height: 900 } }); - const otherReads = boardReads(otherPage); - await openPage(otherPage, origin); - await boardDrawn(otherPage); - - const page = await browser.newPage({ viewport: { width: 1440, height: 900 } }); - const pageReads = boardReads(page); - await openMailbox(page, origin); - await boardDrawn(page); - - const tabs: Tab[] = [ - { leg: "row", name: "tab 1", page, reads: pageReads, loads: 0 }, - { leg: "other-tab", name: "tab 2", page: otherPage, reads: otherReads, loads: 0 }, - ]; - for (const tab of tabs) tab.page.on("load", () => (tab.loads += 1)); - - for (const [i, c] of cases.entries()) { - c.sentAt = await send(page, c.line); - const deadline = c.sentAt + fixture.rowWithinMs; - while (Date.now() < deadline && tabs.some((t) => c.seen.get(t)?.rowAt === undefined)) { - for (const tab of tabs) observe(tab, await readBoard(tab.page)); - await sleep(100); - } - - const name = `case ${i + 1} (${c.token})`; - for (const tab of tabs) { - const seen = c.seen.get(tab) ?? { most: 0 }; - if (seen.rowAt === undefined || seen.rowAt > deadline) { - fail(tab.leg, `${name}: ${tab.name}'s escalations panel showed no row carrying the token within ${fixture.rowWithinMs / 1000}s of Send, with no reload (seen ${secs(c.sentAt, seen.rowAt)})`); - } else if (i > 0) { - const order = seen.order ?? []; - const mine = order.indexOf(c.token); - const earlier = order.indexOf(cases[i - 1]!.token); - if (earlier === -1) { - fail(tab.leg, `${name}: ${tab.name} showed its row without the earlier case's`); - } else if (mine > earlier) { - fail(tab.leg, `${name}: ${tab.name} showed its row below the earlier case's (want newest first)`); - } - } - const window = tab.reads.filter((at) => at >= c.sentAt && at <= (seen.rowAt ?? deadline)).length; - if (window > fixture.maxBoardReads) { - fail("no-poll", `${name}: ${tab.name} read the board ${window} times between Send and the row (at most ${fixture.maxBoardReads})`); - } - evidence.push(`${name}, ${tab.name}: row ${secs(c.sentAt, seen.rowAt)}, ${window} board reads`); - } - - // Let the case settle before the next post, or the idle window, so the - // route is free and nothing is still on its way. Not graded; readings - // meanwhile still count for status and once. - await settled(page, origin, c.token); - for (const tab of tabs) observe(tab, await readBoard(tab.page)); - - if (i === 0) { - const from = Date.now(); - while (Date.now() < from + fixture.idleMs) { - for (const tab of tabs) observe(tab, await readBoard(tab.page)); - await sleep(250); - } - for (const tab of tabs) { - const idle = tab.reads.filter((at) => at >= from).length; - if (idle > 0) { - fail("no-poll", `${tab.name} read the board ${idle} times in ${fixture.idleMs / 1000}s with nothing changing (want none)`); - } - } - evidence.push(`no board reads in either tab in ${fixture.idleMs / 1000}s idle`); - } - } - - // A reload or navigation before the last leg voids the run. - for (const tab of tabs) { - if (tab.loads > 0) fail("void", `${tab.name} reloaded or navigated ${tab.loads} times before the last leg`); - } - for (const c of cases) { - for (const tab of tabs) { - const most = c.seen.get(tab)?.most ?? 0; - if (most > 1) fail("once", `${c.token}: while open, one reading of ${tab.name} showed its row ${most} times`); - } - } - - // ---- once, after the final reload of each tab --------------------------- - for (const tab of tabs) { - await tab.page.reload(); - await tab.page.locator('[data-testid="message-input"]:visible').waitFor({ state: "visible", timeout: 30_000 }); - await boardDrawn(tab.page); - const reading = await readUntil( - () => readBoard(tab.page), - (r) => cases.every((c) => r.rows.some((drawn) => drawn.text.includes(c.token))), - 10_000, - ); - for (const c of cases) { - const count = reading.rows.filter((drawn) => drawn.text.includes(c.token)).length; - if (count !== 1) { - fail("once", `after the reload, ${tab.name}'s escalations panel holds ${count} rows carrying ${c.token} (want 1)`); - } - checkStatus(`${tab.name}, after the reload`, reading, c.token); - } - } - evidence.push("after the reload, each case is one row in each tab"); - } finally { - await browser.close(); - server?.stop(); - } - - if (statusRowsRead === 0) { - fail("status", "no row carrying this run's tokens was drawn in either tab, open or after the reload, so no status could be read"); - } - for (const problem of statusProblems) fail("status", problem); - if (statusRowsRead > 0 && statusProblems.size === 0) { - evidence.push(`every row drawn reads "${fixture.status}", in one list`); - } - - // A control must redden each leg it names, and only those. - if (CONTROL !== "") { - const want = EXPECTED[CONTROL]!; - const legs = new Set(failures.map((f) => /^\[([^\]]+)\]/.exec(f)?.[1] ?? "")); - for (const leg of want) { - if (!legs.has(leg)) failures.push(`[control] GOAL_CONTROL=${CONTROL} left the ${leg} leg green, so that leg cannot fail`); - } - for (const leg of legs) { - if (!want.includes(leg) && leg !== "control") failures.push(`[control] GOAL_CONTROL=${CONTROL} also reddened the ${leg} leg`); - } - } - return { failures, evidence: evidence.join("; ") }; -}); diff --git a/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/fixtures/input.json b/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/fixtures/input.json index 8ae53aa4b2..00a723ccd9 100644 --- a/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/fixtures/input.json +++ b/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/fixtures/input.json @@ -1,6 +1,6 @@ { "port": 3041, - "mailbox": { "kind": "mailbox", "id": "support.help" }, + "coordinator": { "kind": "coordinator", "id": "support.help" }, "replier": { "kind": "agent", "id": "support.devices" }, "route": "[route:support.devices]", "marker": "[scenario:reply-after-a-hold]", diff --git a/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/goal.md b/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/goal.md index 2d1e7650bf..75559971de 100644 --- a/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/goal.md +++ b/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/goal.md @@ -1,30 +1,31 @@ # kitchen-sink-talk › shows the reply without a reload -**Issue:** FIX-1609 (VG of the spec's PLAN; the epic FIX-1592's ER-23); re-pointed by FIX-1611 +**Issue:** FIX-1609 (VG of the spec's PLAN; the epic FIX-1592's ER-23); re-pointed by FIX-1611; converted by FIX-1792 P1 -**Outcome:** With `support.help` open, a person whose post is routed to `support.devices` sees `support.devices is working`, then its line in the mailbox under its own name, with no reload. The same holds in the specialist's own conversation open in another tab. +**Outcome:** With `support.help` open, a person whose post is routed to `support.devices` sees `support.devices is working`, then its line in the person's conversation with `support.help` under its own name, with no reload. The same holds in the specialist's own session for that conversation, open in another tab. **Re-pointed (FIX-1611).** The folder name is kept so its verdict history stays in one place. Until FIX-1611 this read `support.desk`, where every member was woken, and `support.otto` answering there. The mailbox is now routed, so each post names its specialist (`[route:support.devices]`), which the scripted route picks. The second post, sent before the first is answered, goes to the same specialist. The legs are unchanged. -**Input:** `fixtures/input.json`: the mailbox, the specialist that answers there (`support.devices`), what a post carries to be routed to it (`[route:support.devices]`), the scenario marker the scripted model keys on (`[scenario:reply-after-a-hold]`, the reply-in-mailbox answer held about three seconds), the marker it writes on the specialist's line (`[reply:in-mailbox]`), how long a line may take (15 s) and how many snapshot reads the page may make meanwhile (2). Held-out: each run posts three lines, each with a fresh `reply-token-…`, and only those tokens are graded, so a line another run or test left in the shared mailbox can never be the one that passes. The second post goes in while the specialist still works on the first. The third is read in a second surface, the specialist's own conversation for the mailbox, open in a second tab. +**Converted (FIX-1792 P1).** `support.help` is a coordinator worker file on `routing: best-fit`, no longer a mailbox. The post goes from the panel of the person's conversation with it, and the specialist's answer is drawn there under its name; the third post is heard in the specialist's own session for the conversation, a session on the `agent` flow listed in the rail as a run of the conversation (which one is read from the server by the session's `workerId`, as an index, since FIX-1788's rail has no row per specialist). The `main` control goes: no commit has the coordinator without the live view it was taken against, so `no-live` alone carries that red state. + +**Input:** `fixtures/input.json`: the coordinator, the specialist that answers there (`support.devices`), what a post carries to be routed to it (`[route:support.devices]`), the scenario marker the scripted model keys on (`[scenario:reply-after-a-hold]`, the reply-in-mailbox answer held about three seconds), the marker it writes on the specialist's line (`[reply:in-mailbox]`), how long a line may take (15 s) and how many snapshot reads the page may make meanwhile (2). Held-out: each run posts three lines, each with a fresh `reply-token-…`, and only those tokens are graded, so a line another run or test left behind can never be the one that passes. The second post goes in while the specialist still works on the first. The third is read in a second surface, the specialist's own session for the conversation, open in a second tab. **Signal:** one real browser against kitchen-sink's **production build** (built by the run, never assumed), served by `next start` on the scripted model, the in-memory store and **no model key**. Everything graded is read off the open page; the page is reloaded once, for the last leg only. -- **working**: for each of the first two posts, the mailbox's panel shows `support.devices is working` in a reading before the one that first shows its line for that post, and before the next post is sent: a Send re-reads the runs even on a page that doesn't follow its session, so a later reading could show the row without the stream. The specialist working on the first post shows the same row, so for the second the reading must also come after its line for the first is in: still working on the second once it has answered the first. -- **line**: for each of the first two posts, within 15 s of Send, the open panel shows the specialist's line carrying the token and `[reply:in-mailbox]`, labelled `support.devices` (a second copy is graded under **once**); once its last line lands, the working row is gone within 5 s. For the third post, the specialist's conversation open in the second tab shows the post heard, and its answer after it, within 15 s. +- **working**: for each of the first two posts, the conversation's panel shows `support.devices is working` in a reading before the one that first shows its line for that post, and before the next post is sent: a Send re-reads the runs even on a page that doesn't follow its session, so a later reading could show the row without the stream. The specialist working on the first post shows the same row, so for the second the reading must also come after its line for the first is in: still working on the second once it has answered the first. +- **line**: for each of the first two posts, within 15 s of Send, the open panel shows the specialist's line carrying the token and `[reply:in-mailbox]`, labelled `support.devices` (a second copy is graded under **once**); once its last line lands, the working row is gone within 5 s. For the third post, the specialist's session open in the second tab shows the post heard, and its answer after it, within 15 s. - **once**: no reading while the page is open shows a post or the specialist's line twice, and after the one reload each post and each reply shows exactly once. - **no-poll**: between Send and the line, the page reads a session snapshot (`GET …/sessions//state`) at most twice. The same holds for the second tab and the third post. -**Anti-game:** no reload before **once**. Nothing asserted on the wire, a hook's return, a package test or a CLI run: the page's own requests are only counted, for **no-poll**, and the server's reads behind the stream are not the page's. Before the reload the run reads the mailbox's state until every answer is kept, so the reload grades what was kept and not how fast; that read is not graded. Only this run's tokens count. +**Anti-game:** no reload before **once**. Nothing asserted on the wire, a hook's return, a package test or a CLI run: the page's own requests are only counted, for **no-poll**, and the server's reads behind the stream are not the page's. Before the reload the run reads the conversation's state until every answer is kept, so the reload grades what was kept and not how fast; that read is not graded. Only this run's tokens count. -**Model:** n/a. kitchen-sink's scripted model routes each post by the specialist it names and answers the seats (epic FIX-1592 D3): after a three-second hold, a step calling `post-to-mailbox` with no text, so the real tool runs, then a text step. Keyless: the server runs with `AI_GATEWAY_API_KEY` empty. +**Model:** n/a. kitchen-sink's scripted model scripts best fit's evaluation by the specialist each post names, and answers the specialist (epic FIX-1592 D3) after a three-second hold. Keyless: the server runs with `AI_GATEWAY_API_KEY` empty. **Run:** `PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/shows-the-reply-without-a-reload/run.mts` **Controls:** on the same command. - `GOAL_CONTROL=no-live`: the page is opened with `?goalControl=no-live`, which a test-mode build honours (`apps/kitchen-sink/lib/goal-control.ts`, `pageGoalControl`) by leaving the rail's panels without `live`. The page stays silent until the reload. Must FAIL at **working** and **line**, and at nothing else. -- `GOAL_CONTROL=main`: the app before the live view, run from a checkout of it with this directory copied in (for FIX-1609, `main`; for the FIX-1611 re-point, the FIX-1611 branch before FIX-1609 merged into it, since `main` had no `support.help`), and the scripted seat that holds (`apps/kitchen-sink/lib/e2e-mock-script.ts` and `apps/kitchen-sink/test/mock-flowstate.ts`), so the control differs from the change only in the stream and the panels. The app knows no control by that name; it tells the run which legs must fail. Must FAIL at **working** and **line**, and at nothing else. ## Verdict log | Date | Commit | Model | Verdict | Notes | @@ -46,3 +47,6 @@ | 2026-09-28 | FIX-1611 branch merged with `main` at b11024b42 (FIX-1609 merged) | scripted | FAIL (control) | **Re-pointed.** `GOAL_CONTROL=no-live` on the same tree: failed at **working** (posts 1 and 2) and **line** (posts 1, 2 and 3) and nothing else. **once** and **no-poll** green. | | 2026-09-28 | FIX-1622 branch at 91a5afdc4 (the team panel's board also follows `support.help`) | scripted | **PASS** | Neighbour re-run. Post 1: working +1.0s, line +4.0s under `support.devices`, 2 snapshot reads. Post 2: working +2.9s, line +3.9s, 1 snapshot read. The working row went 0.1s after the last line. Post 3: heard and answered in the second tab +4.0s, 0 snapshot reads. After the reload each post and its reply shows once. | | 2026-09-28 | same | scripted | FAIL (control) | `GOAL_CONTROL=no-live`: failed at **working** (posts 1 and 2) and **line** (posts 1, 2 and 3) and nothing else. **once** and **no-poll** green. | +| 2026-10-09 | `main` at 57d2c10e9 | scripted | FAIL | **Taken first, on clean `main`, before the conversion.** The previous check fails at setup: its second tab opens a rail row named for the specialist, and since FIX-1788 P4 the rail lists one `agent` copy and no row per worker. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Converted.** Fresh production build, keyless. Post 1: working +0.9s, line +4.0s under `support.devices`, 2 snapshot reads. Post 2: working +3.0s, line +4.0s, 1 snapshot read. The working row went 0.1s after the last line. Post 3 heard and answered in `support.devices`' open session `dsx_…` +3.9s, 0 snapshot reads. After the reload each post and its reply show once. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-live`: failed at **working** (posts 1 and 2) and **line** (posts 1, 2 and 3) and nothing else. **once** and **no-poll** green. | diff --git a/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/run.mts b/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/run.mts index f48c5d1ff5..d5a4c5dff9 100644 --- a/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/run.mts +++ b/goals/kitchen-sink-talk/shows-the-reply-without-a-reload/run.mts @@ -10,12 +10,13 @@ * (`[route:support.devices]`), which the scripted route picks; the second, * sent before the first is answered, goes to the same specialist. The * scripted specialist holds its answer about three seconds, so "working" has - * time to show. Two posts from the mailbox's panel, the second while the - * specialist works on the first; then a third, with the specialist's own - * conversation for the mailbox open in a second tab. The page - * is never reloaded until the last leg. Four legs, graded per post: + * time to show. Two posts from the panel of the person's conversation with + * `support.help`, the second while the specialist works on the first; then a + * third, with the specialist's own session for that conversation open in a + * second tab. The page is never reloaded until the last leg. Four legs, + * graded per post: * - * working `support.devices is working` shows in the mailbox's panel + * working `support.devices is working` shows in the conversation's panel * before its line for the post does, and before the next post is sent * (a Send re-reads the runs on any page); for the second post, * after its line for the first is in, since the row is the same. @@ -24,18 +25,18 @@ * `support.devices` * (a second copy is graded under once); the working row is gone * once its run ends. - * For the third post, the specialist's open conversation shows the + * For the third post, the specialist's open session shows the * post heard and its answer after it, in the second tab. * once no line shows twice while the page is open, and after the one * reload each post and each reply shows exactly once. * no-poll at most two snapshot reads by the page between Send and the line. * * Everything graded is read off the page. The server is read only to know - * when to reload, and nothing it says is graded. + * when to reload and which of the specialist's sessions to open, and nothing + * it says is graded. * * Run: PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/shows-the-reply-without-a-reload/run.mts * Controls: GOAL_CONTROL=no-live (must FAIL at working and line, and nothing else) - * GOAL_CONTROL=main (the app before the live view, from a checkout of it: the same) */ import { randomUUID } from "node:crypto"; import type { Page } from "playwright"; @@ -43,12 +44,13 @@ import { loadFixture, runGoal } from "../../lib/index.mts"; import { buildKitchenSink, conversation, - open, + coordinatorConversation, + openWorkerCopy, panel, - rail, readUntil, - row, + showCoordinator, startKitchenSink, + workerSessions, type KitchenSinkServer, } from "../../lib/kitchen-sink.mts"; import { launchChromium } from "../../lib/playwright.mts"; @@ -60,7 +62,7 @@ interface Seat { interface Fixture { port: number; - mailbox: Seat; + coordinator: Seat; replier: Seat; /** What a post carries to be routed to `replier`. */ route: string; @@ -78,10 +80,6 @@ const EXPECTED: Record = { // The panels don't ask to follow their session: the page stays silent // until a reload, so "working" never shows and neither does the line. "no-live": ["working", "line"], - // The app before the live view, run from a checkout of it with these files - // copied in. The app knows no control by this name; it tells this run which - // legs must fail. - main: ["working", "line"], }; if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); @@ -102,12 +100,10 @@ async function openPage(page: Page, origin: string): Promise { await page.locator('[data-testid="message-input"]:visible').waitFor({ state: "visible", timeout: 30_000 }); } -/** Open the mailbox's panel. */ -async function openMailbox(page: Page, origin: string): Promise { +/** Open the panel of the person's conversation with the coordinator. */ +async function openHelpDesk(page: Page, origin: string): Promise { await openPage(page, origin); - await open(page, fixture.mailbox.kind); - await row(page, fixture.mailbox.id).click(); - await panel(page).getByTestId("mailbox-transcript").waitFor({ timeout: 15_000 }); + await showCoordinator(page, fixture.coordinator.id); } /** When the page read a session's snapshot (`GET …/sessions//state`). */ @@ -125,14 +121,14 @@ interface Line { text: string; } -/** The mailbox's panel as drawn: whether the specialist shows as working, and every line. */ -async function readMailbox(page: Page): Promise<{ at: number; working: boolean; lines: Line[] }> { +/** The conversation's panel as drawn: whether the specialist shows as working, and every line. */ +async function readHelpDesk(page: Page): Promise<{ at: number; working: boolean; lines: Line[] }> { const drawn = panel(page); const [working, lines] = await Promise.all([ drawn.getByTestId("working-row").filter({ hasText: `${fixture.replier.id} is working` }).count(), - drawn.getByTestId("mailbox-line").evaluateAll((els) => + drawn.getByTestId("coordinator-line").evaluateAll((els) => els.map((el) => ({ - label: el.querySelector('[data-testid="mailbox-line-label"]')?.textContent ?? "", + label: el.querySelector('[data-testid="coordinator-line-label"]')?.textContent ?? "", text: el.textContent ?? "", })), ), @@ -160,7 +156,7 @@ const postsOf = (lines: Line[], token: string) => lines.filter((l) => l.text.includes(token) && !l.text.includes(fixture.lineMarker)); /** - * Fold one reading of the mailbox into what each sent post has seen so far. + * Fold one reading of the conversation into what each sent post has seen so far. * * A reading counts as the specialist working on a post only when nothing else * could have put the row there. The specialist working on an earlier post @@ -192,11 +188,11 @@ function observe(posts: Post[], reading: { at: number; working: boolean; lines: } } -/** Post a line from the mailbox's panel once its composer is free, reading the panel meanwhile. */ +/** Post a line from the conversation's panel once its composer is free, reading the panel meanwhile. */ async function send(page: Page, posts: Post[], post: Post): Promise { - const box = panel(page).getByLabel("Post to this mailbox"); + const box = panel(page).getByLabel("Post to this coordinator"); for (let waited = 0; waited < 10_000; waited += 100) { - observe(posts, await readMailbox(page)); + observe(posts, await readHelpDesk(page)); if ((await box.isEnabled()) && (await box.inputValue()) === "") break; await sleep(100); } @@ -233,28 +229,28 @@ await runGoal(async () => { const origin = server.origin; const page = await browser.newPage({ viewport: { width: 1440, height: 900 } }); const reads = snapshotReads(page); - await openMailbox(page, origin); + await openHelpDesk(page, origin); // ---- two posts, the second while the specialist works on the first ----- await send(page, posts, first); - for (let r = await readMailbox(page); ; r = await readMailbox(page)) { + for (let r = await readHelpDesk(page); ; r = await readHelpDesk(page)) { observe(posts, r); if (first.workingAt !== undefined || first.lineAt !== undefined || Date.now() - first.sentAt > 2_000) break; await sleep(100); } await send(page, posts, second); - const mailboxPosts = [first, second]; + const deskPosts = [first, second]; const lastDeadline = second.sentAt + fixture.lineWithinMs; - while (Date.now() < lastDeadline && mailboxPosts.some((p) => p.lineAt === undefined)) { - observe(posts, await readMailbox(page)); + while (Date.now() < lastDeadline && deskPosts.some((p) => p.lineAt === undefined)) { + observe(posts, await readHelpDesk(page)); await sleep(100); } // The specialist's run ends right after its line; give the row a few seconds to go. let clearedAt: number | undefined; - const lastLine = Math.max(...mailboxPosts.map((p) => p.lineAt ?? 0)); - if (mailboxPosts.every((p) => p.lineAt !== undefined)) { + const lastLine = Math.max(...deskPosts.map((p) => p.lineAt ?? 0)); + if (deskPosts.every((p) => p.lineAt !== undefined)) { while (Date.now() < lastLine + 5_000) { - const r = await readMailbox(page); + const r = await readHelpDesk(page); observe(posts, r); if (!r.working) { clearedAt = r.at; @@ -264,7 +260,7 @@ await runGoal(async () => { } } - for (const [i, post] of mailboxPosts.entries()) { + for (const [i, post] of deskPosts.entries()) { const name = `post ${i + 1} (${post.token})`; if (post.workingAt === undefined) { const after = i === 0 ? ", before the next post was sent" : `, once ${fixture.replier.id} had answered the earlier post`; @@ -281,7 +277,7 @@ await runGoal(async () => { } evidence.push(`${name}: working ${secs(post.sentAt, post.workingAt)}, line ${secs(post.sentAt, post.lineAt)} under ${post.label ?? "nobody"}, ${window} snapshot reads`); } - if (mailboxPosts.every((p) => p.lineAt !== undefined)) { + if (deskPosts.every((p) => p.lineAt !== undefined)) { if (clearedAt === undefined) { fail("line", `"${fixture.replier.id} is working" still showed 5s after its last line`); } else { @@ -289,17 +285,23 @@ await runGoal(async () => { } } - // ---- a third post, heard in the specialist's own conversation in a second tab + // ---- a third post, heard in the specialist's own session in a second tab + // Which of the specialist's sessions the conversation's posts run in, from + // the server's index by worker; the page must list it as a run of the + // conversation. Neither read is graded. + const helpConversation = await coordinatorConversation(page, origin, fixture.coordinator.id); + const [delegateRun] = (await workerSessions(page, origin, fixture.replier.id, { runs: true })).filter( + (s) => helpConversation !== undefined && s.parentSessionId === helpConversation, + ); const seatPage = await browser.newPage({ viewport: { width: 1440, height: 900 } }); const seatReads = snapshotReads(seatPage); await openPage(seatPage, origin); - await open(seatPage, fixture.replier.kind); - await open(seatPage, fixture.replier.id); - const runOfMailbox = rail(seatPage) - .locator(`ul[data-leaf="${fixture.replier.id}"]`) - .locator(`[data-dispatch-run-of="${fixture.mailbox.id}"]`); - await runOfMailbox.first().waitFor({ timeout: 15_000 }); - await runOfMailbox.first().click(); + if (delegateRun === undefined) throw new Error(`${fixture.replier.id} has no session for the conversation with ${fixture.coordinator.id}`); + const runOfConversation = (await openWorkerCopy(seatPage)).locator( + `[data-session-id="${delegateRun.id}"][data-dispatch-run-of="${helpConversation}"]`, + ); + await runOfConversation.waitFor({ timeout: 15_000 }); + await runOfConversation.click(); await readUntil(() => conversation(seatPage), (ms) => ms.length > 0, 10_000); await send(page, posts, third); @@ -315,12 +317,12 @@ await runGoal(async () => { } const seatWindow = seatReads.filter((at) => at >= third.sentAt && at <= (heardAt ?? third.sentAt + fixture.lineWithinMs)).length; if (heardAt === undefined) { - fail("line", `post 3 (${third.token}): ${fixture.replier.id}'s open conversation never showed the post heard and its answer, with no reload`); + fail("line", `post 3 (${third.token}): ${fixture.replier.id}'s open session never showed the post heard and its answer, with no reload`); } if (seatWindow > fixture.maxSnapshotReads) { fail("no-poll", `post 3 (${third.token}): the second tab read a snapshot ${seatWindow} times between Send and ${fixture.replier.id}'s answer (at most ${fixture.maxSnapshotReads})`); } - evidence.push(`post 3 (${third.token}): heard and answered in ${fixture.replier.id}'s open conversation ${secs(third.sentAt, heardAt)}, ${seatWindow} snapshot reads`); + evidence.push(`post 3 (${third.token}): heard and answered in ${fixture.replier.id}'s open session ${delegateRun.id} ${secs(third.sentAt, heardAt)}, ${seatWindow} snapshot reads`); // ---- once: nothing twice while open, then each line once after a reload for (const post of posts) { @@ -333,7 +335,7 @@ await runGoal(async () => { await readUntil( async () => { const res = await page.request.get( - `${origin}/api/flows/sessions/${fixture.mailbox.id}/state?include_items=true&item_types=component&limit=1000`, + `${origin}/api/flows/sessions/${helpConversation}/state?include_items=true&item_types=message&limit=1000`, ); const text = await res.text(); return posts.every((p) => text.split(p.token).length - 1 >= 2); @@ -342,9 +344,9 @@ await runGoal(async () => { 20_000, ); await page.reload(); - await openMailbox(page, origin); + await openHelpDesk(page, origin); const drawn = await readUntil( - async () => (await readMailbox(page)).lines, + async () => (await readHelpDesk(page)).lines, (ls) => posts.every((p) => replies(ls, p.token).length > 0), 10_000, ); diff --git a/goals/lib/kitchen-sink.mts b/goals/lib/kitchen-sink.mts index fcefa24b12..55f15d3d72 100644 --- a/goals/lib/kitchen-sink.mts +++ b/goals/lib/kitchen-sink.mts @@ -127,16 +127,88 @@ export async function open(page: Page, name: string): Promise { if ((await button.getAttribute("aria-expanded")) !== "true") await button.click(); } -/** Press "New conversation" on a seat's row and wait for the panel. */ -export async function newConversation(page: Page, seat: string): Promise { - await rail(page) - .locator(`[data-instance-id="${seat}"]`) - .locator("xpath=..") - .getByRole("button", { name: "New conversation" }) - .click(); +/** + * Press "Talk" on a worker's row in the roster panel, which opens the person's + * conversation with that worker (found or started), and wait for the panel. + */ +export async function talkTo(page: Page, worker: string): Promise { + await page.locator(`[data-testid="roster"]:visible [data-worker-id="${worker}"]`).getByTestId("roster-talk").click(); await panel(page).waitFor({ timeout: 15_000 }); } +/** The flow every specialist runs on, registered as one copy at its kind. */ +export const WORKER_FLOW = "agent"; + +/** The coordinator flow's kind, whose rail row opens into the person's conversation with each coordinator. */ +export const COORDINATOR_FLOW = "coordinator"; + +/** + * Open the worker flow's kind and its one copy in the rail, unless already + * open, and wait for the copy's list to load. Every worker's sessions are + * listed there, each one a session that names its worker. + */ +export async function openWorkerCopy(page: Page): Promise { + for (const button of [ + rail(page).locator(`button[data-kind="${WORKER_FLOW}"]`), + rail(page).locator(`button[data-instance-id="${WORKER_FLOW}"]`), + ]) { + await button.waitFor({ timeout: 15_000 }); + if ((await button.getAttribute("aria-expanded")) !== "true") await button.click(); + } + const leaf = rail(page).locator(`ul[data-leaf="${WORKER_FLOW}"]`); + await leaf.waitFor({ timeout: 15_000 }); + await readUntil( + async () => (await leaf.locator("[data-session-id]").count()) + (await leaf.getByText("No sessions yet").count()), + (n) => n > 0, + 10_000, + ); + return leaf; +} + +/** Show the person's conversation with `coordinator` in the panel, from the rail. */ +export async function showCoordinator(page: Page, coordinator: string): Promise { + await open(page, COORDINATOR_FLOW); + await row(page, coordinator).click(); + await panel(page).getByTestId("coordinator-transcript").waitFor({ timeout: 15_000 }); +} + +/** One of a worker's sessions, as the server lists it. */ +export interface WorkerSession { + id: string; + parentSessionId?: string | null; +} + +/** + * The person's sessions with `worker` on `flow`, as the server lists them by + * the session's readonly `workerId`; with `runs`, the sessions a coordinator's + * post started for it too. An index, never graded: what each session holds is + * read on the page. + */ +export async function workerSessions( + page: Page, + origin: string, + worker: string, + options: { runs?: boolean; flow?: string; user?: string } = {}, +): Promise { + const query = new URLSearchParams({ + flowId: options.flow ?? WORKER_FLOW, + userId: options.user ?? "devuser", + "state.workerId": worker, + limit: "100", + ...(options.runs === true ? { include: "dispatch-runs" } : {}), + }); + const res = await page.request.get(`${origin}/api/flows/sessions?${query}`); + if (!res.ok()) throw new Error(`the sessions of ${worker} could not be listed: ${res.status()} ${await res.text()}`); + return ((await res.json()) as { sessions: WorkerSession[] }).sessions; +} + +/** The person's conversation with `coordinator`: the one session the rail opens. An index, never graded. */ +export async function coordinatorConversation(page: Page, origin: string, coordinator: string): Promise { + const sessions = await workerSessions(page, origin, coordinator, { flow: COORDINATOR_FLOW }); + if (sessions.length > 1) throw new Error(`the person has ${sessions.length} conversations with ${coordinator} (want at most 1)`); + return sessions[0]?.id; +} + /** Poll `read` until `done` holds, or the time is up; return the last reading either way. */ export async function readUntil(read: () => Promise, done: (value: T) => boolean, ms = 15_000): Promise { let value = await read(); diff --git a/goals/pentest-lab/a-post-reaches-both-declared-seats/fixtures/input.json b/goals/pentest-lab/a-post-reaches-both-declared-seats/fixtures/input.json index cc5d107824..a71aa6b55b 100644 --- a/goals/pentest-lab/a-post-reaches-both-declared-seats/fixtures/input.json +++ b/goals/pentest-lab/a-post-reaches-both-declared-seats/fixtures/input.json @@ -1,8 +1,8 @@ { "post": "Sweep the target list and report what answered.", - "mailbox": { + "coordinator": { "id": "pentest.findings", - "members": ["pentest.recon", "pentest.triage"], + "delegates": ["pentest.recon", "pentest.triage"], "charterToken": "CHARTER-TOKEN-C4D28" }, "documents": { @@ -52,16 +52,16 @@ "SKILL-ORG-FORMAT-2B7C4": "org/skills/report-format/SKILL.md", "SCOPE-TOKEN-5D0E9": "teams/pentest/resources/scope.md", "SKILL-PENTEST-SCAN-91AF6": "teams/pentest/skills/port-scan/SKILL.md", - "CHARTER-TOKEN-C4D28": "teams/pentest/mailboxes/findings/MAILBOX.md", + "CHARTER-TOKEN-C4D28": "teams/pentest/workers/findings/WORKER.md", "WORKER-RECON-7E2B1": "teams/pentest/workers/recon/WORKER.md", "SKILL-RECON-SWEEP-A03D5": "teams/pentest/workers/recon/skills/sweep/SKILL.md", "WORKER-TRIAGE-D6C83": "teams/pentest/workers/triage/WORKER.md", "WORKER-SCRIBE-B19F7": "teams/audit/workers/scribe/WORKER.md", "SKILL-AUDIT-SCAN-3C8E2": "teams/audit/skills/port-scan/SKILL.md" }, - "members": ["pentest.recon", "pentest.triage"], + "delegates": ["pentest.recon", "pentest.triage"], "silentSeat": "audit.scribe", - "nonMemberAuthor": "audit.scribe", + "claimedAuthor": "audit.scribe", "refusalTrees": { "unknownKind": { "dir": "unknown-kind", @@ -78,10 +78,10 @@ "otherWorker": "pentest.triage" } }, - "unknownMemberScenario": { + "unknownDelegateScenario": { "dir": "unknown-member", - "mailboxId": "pentest.briefing", - "unknownMember": "pentest.nobody" + "coordinatorId": "pentest.briefing", + "unknownDelegate": "pentest.nobody" }, "wait": { "timeoutMs": 30000, diff --git a/goals/pentest-lab/a-post-reaches-both-declared-seats/goal.md b/goals/pentest-lab/a-post-reaches-both-declared-seats/goal.md index 3ac2184ba3..7b9c2a3d57 100644 --- a/goals/pentest-lab/a-post-reaches-both-declared-seats/goal.md +++ b/goals/pentest-lab/a-post-reaches-both-declared-seats/goal.md @@ -1,49 +1,73 @@ # pentest-lab › it delivers a post to both declared seats -**Issue:** FIX-1355 (the pentest lab — the four file conventions, proved together; epic FIX-1351) +**Issue:** FIX-1355 (the pentest lab — the four file conventions, proved together; epic FIX-1351); rewritten by FIX-1792 P1 -**Outcome:** Somebody writes ten Markdown files — an org level and two teams holding a mailbox, two documents, five skill folders and three worker files — and points one call at the folder. Three seats are hired, a mailbox opens at the id its two folder names mint, and a single operator post into that mailbox wakes both declared members through the framework's own fan-out. Each answers using the document, the skills and the instructions its own folders gave it, and both answers land in one transcript. A third seat, in another team, holding a skill of the same name as one of theirs, is never woken and appears nowhere. The transcript reads the same after the store is closed and the host rebuilt. +**Outcome:** Somebody writes ten Markdown files — an org level and two teams holding a coordinator, two documents, five skill folders and three seat files — and points one call at the folder. Three seats and one coordinator are hired, the coordinator is a worker file at the id its two folder names mint, and a single operator post to it reaches both declared delegates through Workforce's own coordinator. Each answers using the document, the skills and the instructions its own folders gave it, and both answers land in the operator's conversation, each under its seat's name. A third seat, in another team, holding a skill of the same name as one of theirs, is never handed the post and appears nowhere. The conversation reads the same after the store is closed and the host rebuilt. + +**Rewritten (FIX-1792 P1).** The folder name and the rule numbers are kept so the verdict history stays in one place. Until FIX-1792 the tree held a mailbox, `teams/pentest/mailboxes/findings/MAILBOX.md`, and a notify block of the lab's own woke its members. It is now `teams/pentest/workers/findings/WORKER.md`, a coordinator on `routing: everyone` with the same id, description, body and delegates, and the lab registers Workforce's `coordinator` flow instead of a notify block. What each rule now reads: + +- **BR-1, BR-5**: the root produces exactly one coordinator. Its delegates are read through its own `listDelegates` on the operator's conversation. Its charter is its `WORKER.md` body, which is a coordinator's instructions. A coordinator that routes to everyone runs no turn of its own, so nothing reads the body at run time. It is read where it last goes, the worker the installation hands the `coordinator` flow. +- **BR-6 to BR-14**: who the post reached is the coordinator's own routing record (`coordinator-route`), not a log the lab kept. An answer is a `message` drawn under the delegate the delivery named. The answer itself carries no author, so the old `authorVerified: false` clause has nothing to read and goes. +- **BR-8**: the cycle break is `rounds: 0`, the coordinator's default. An answer is routed to nobody, so the conversation holds one routing decision, the post's. The old second lab, where a member's own line was posted by hand, goes: the operator's door takes no author (BR-17), so there is no such line to post. Its control, `answers-go-on`, now acts on the main path. +- **BR-9**: a declared delegate the coordinator can't reach is skipped with why on the routing record, while the other still answers. "Can't reach" is a delegate whose flow takes no posts, the gate's stand-in for the old member with no address. +- **BR-10** (FIX-1792's BR-10, which replaces FIX-1355's): a coordinator whose `delegates:` names a worker no file declares is **refused at load**, naming both. The old rule opened the mailbox and skipped the delivery. Its opposite state is the same coordinator with the unknown delegate taken off, which opens, and its one known delegate answers. +- **BR-17**: the operator's door takes `{ message }` and nothing else, so any author claim is refused naming the claim and writes nothing. The same post claiming nothing is accepted. A declared member's claim is no longer accepted, since there is no author field to claim with. This is the **contract gate** for the epic's ER-19, ER-20 and ER-21. Its model-backed half is the sibling goal, `a-seat-answers-from-its-own-document`, which drives the same tree, the same hire and the same wiring with a generator in one slot. -**Input:** `fixtures/input.json` — every seat id, document ref, skill name, held-out token and expected union. The tree itself is `../lab/workforce/`, ten convention files; the two refusal trees and their corrected twins are `../lab/refusal-trees/`, and the unknown-member mailbox is `../lab/scenario-trees/`. Held-out in the strict sense: **each token appears in exactly one convention file and nowhere in the lab's code or in the operator's post**, and leg 0 scans the whole tree and the lab's `.mts` files (including `workforce/flows/workers/probe.mts`) to prove it before anything is built. Swapping a token, a skill folder or a document for another valid one must still pass a correct implementation. +**Input:** `fixtures/input.json` — every seat id, document ref, skill name, held-out token and expected union, and the coordinator's id, delegates and charter token. The tree itself is `../lab/workforce/`, ten convention files. The two refusal trees and their corrected twins are `../lab/refusal-trees/`, and the coordinator naming an unknown delegate is `../lab/scenario-trees/`. Held-out in the strict sense: **each token appears in exactly one convention file and nowhere in the lab's code or in the operator's post**, and leg 0 scans the whole tree and the lab's `.mts` files (including `workforce/flows/workers/probe.mts`) to prove it before anything is built. Swapping a token, a skill folder or a document for another valid one must still pass a correct implementation. The one deliberate inversion: **the `scope-check` tool's clearance value is minted in code at construction and must appear in no file**, because that leg's claim is the opposite one — a marker a *file* could have supplied would prove nothing was called. It is graded by the model-backed sibling, which is where a tool slot exists. -**Signal:** ten legs over the real path — the loaders, `hireWorkforce`, `mailboxInstances`, `openMailboxes` through the real session routes, and `runAction` — with on-disk SQLite stores. +**Signal:** ten legs over the real path — the loaders, `createWorkerInstallation`, `hireWorkforce`, the session routes that open the operator's conversation, the coordinator's own door, and `runAction` — with on-disk SQLite stores. 0. Every held-out token lives in exactly one convention file, in none of the lab's own code, and in no part of the post. -1. **BR-1, BR-5** — one root produces three seats, two documents and **exactly one** mailbox — the complete id set, not its first element — at `pentest.findings`, carrying its two declared members **and its `MAILBOX.md` body as its charter**, read off the opened session's own state. Every seat is on the lab's own kind and requires an org. -2. **BR-3** — each seat's **exact expected skill union** (set equality, not presence and not sibling-exclusion), read off the seat itself through its own config inside a running block. Each skill carries its own level's body token; no seat carries another team's same-named copy. The silent seat included — the transcript can never carry it. -3. **BR-6, BR-7, BR-11, BR-12, BR-14** — one post, no author. The fan-out addresses exactly the two members; both lines are in the final transcript, stored `authorVerified: false`, each carrying its own instructions token, its own document token and its exact skill union, and neither sibling's instructions or document. -4. **BR-8, BR-13** — the transcript settles at three lines and stays there: each member's own post-back reached the router and woke nobody. The fan-out's **addressed list** shows the third seat was never dispatched to, and its tokens are nowhere in the transcript. The author gate gets its **own lab** besides, where one already-authored line is posted by hand and the router is observed recording it and dispatching to nobody — post-backs suppressed in **both** halves, so the suppression cannot be what makes the control red. +1. **BR-1, BR-5** — one root produces three seats, two documents and **exactly one** coordinator — the complete id set, not its first element — at `pentest.findings`, carrying its two declared delegates (read off the conversation) **and its `WORKER.md` body as its charter**. Every seat is on the lab's own kind and requires an org. +2. **BR-3** — each seat's **exact expected skill union** (set equality, not presence and not sibling-exclusion), read off the seat itself through its own config inside a running block. Each skill carries its own level's body token; no seat carries another team's same-named copy. The silent seat included — the conversation can never carry it. +3. **BR-6, BR-7, BR-11, BR-12, BR-14** — one post, no author. The coordinator routes it once, to everyone, and delivers it to exactly the two delegates; both answers are in the conversation, each drawn under its delegate as an `assistant` message, each carrying its own instructions token, its own document token and its exact skill union, and neither sibling's. +4. **BR-8, BR-13** — the conversation settles at three lines and stays there: the routing record shows no decision after the post's, so no answer woke anyone. The routing record and the delegate sessions show the third seat was never handed the post, and its tokens are nowhere in the conversation. 5. **BR-16** — a request carrying no org is refused `OrgRequired` at `runAction`, which is the door every action passes through, while the same read with an org lands. -6. **BR-15** — the store is closed and a fresh host built over the same file; the transcript reads the same, line for line, comparing the **complete** stored records (id, timestamp, principal, author, `authorVerified`, body) and normalising only order. -7. **BR-9, BR-10, BR-17** — a declared member with no address is recorded and skipped while the other still answers; a `MAILBOX.md` naming a member that is no hired seat still opens, keeps it on the roster and skips its delivery; a post claiming a non-member author is refused `author-not-a-member` and writes nothing, while a declared member's claim is accepted. +6. **BR-15** — the store is closed and a fresh host built over the same file; the conversation reads the same, line for line, comparing the **complete** stored message items and normalising only order. +7. **BR-9, BR-10, BR-17** — a declared delegate whose flow takes no posts is skipped with why while the other still answers; a coordinator naming a delegate no file declares is refused at load naming both, and opens once that delegate is taken off; a post claiming an author is refused and writes nothing, while the same post claiming none is accepted. 8. **BR-18** — a `WORKER.md` naming an unregistered kind, and one naming a `tools:` key the catalog does not carry, each refuse the **whole** hire naming that worker, with nothing hired; each tree, corrected in that one key, hires both seats cleanly. -9. **BR-19** — both bounded helpers, both halves. The transcript wait fails loudly on a short transcript, naming how many lines it saw, and returns on a target the run reaches; the settle wait **raises** on a bound it cannot satisfy rather than returning the last count it saw, and returns on a transcript that has gone quiet. +9. **BR-19** — both bounded helpers, both halves. The conversation wait fails loudly on a short conversation, naming how many lines it saw, and returns on a target the run reaches; the settle wait **raises** on a bound it cannot satisfy rather than returning the last count it saw, and returns on a conversation that has gone quiet. -**Anti-game:** a hollow pass would assert that the hires returned, the mailbox opened and three lines exist — all true of a lab where both seats share one settings bag, or where nothing was really wired and every isolation clause is vacuous. *A fixture that fails to wire the thing under test produces a pass, not an error.* So the check: +**Anti-game:** a hollow pass would assert that the hires returned, the conversation opened and three lines exist — all true of a lab where both seats share one settings bag, or where nothing was really wired and every isolation clause is vacuous. *A fixture that fails to wire the thing under test produces a pass, not an error.* So the check: - MUST assert what each seat **carries** before asserting what it must not. "It cannot read its sibling's document" is trivially true of a seat that can read nothing. - MUST grade a seat's skills as **set equality against its exact expected union**, never as presence and never as "none of its sibling's". The two pentest seats are one team and legitimately share the org and team folders; a sibling-exclusion there fails a correct run, and presence alone misses a leak. Isolation is **across teams**, graded on the two `port-scan` folders' **bodies**, which is the only thing that differs between them. -- MUST read every value back through the real route — the seat's own config inside a running block, the resource surface, the mailbox's own `read` — never off `hireWorkforce`'s return value, which will agree with itself. -- MUST grade the silent seat's union **off the seat**, because BR-13 forbids it from putting anything in the transcript, and a rule with no evidence path is what this epic exists to refuse. -- MUST prove the third seat was never **dispatched to**, not merely that it is absent from the transcript. A stray delivery whose seat produces no post is otherwise indistinguishable from no delivery. -- MUST NOT assert transcript **order**. Two concurrent appends have none, and asserting one is the fastest way to write a check that fails on correct code. -- MUST pair every "did not happen" with its opposite state — the skipped member, once addressed, lands; the refused author, once a member, is accepted; the org-less read, once given an org, succeeds. Either half alone is satisfied by a router or a door that does nothing. +- MUST read every value back through the real route — the seat's own config inside a running block, the resource surface, the coordinator's own `listDelegates` and routing record, the session's state route — never off `hireWorkforce`'s return value, which will agree with itself. The charter is the one exception, said out loud above: nothing reads it at run time on this routing. +- MUST grade the silent seat's union **off the seat**, because BR-13 forbids it from putting anything in the conversation, and a rule with no evidence path is what this epic exists to refuse. +- MUST prove the third seat was never **handed the post**, not merely that it is absent from the conversation. A stray delivery whose seat produces no answer is otherwise indistinguishable from no delivery. +- MUST NOT assert conversation **order**. Two concurrent answers have none, and asserting one is the fastest way to write a check that fails on correct code. +- MUST pair every "did not happen" with its opposite state — the skipped delegate, on the lab's kind, lands; the refused coordinator, without its unknown delegate, opens; the refused author claim, dropped, is accepted; the org-less read, once given an org, succeeds. Either half alone is satisfied by a coordinator or a door that does nothing. - MUST assert BR-18 left **nothing** hired, and MUST run each bad tree's corrected twin, or the refusal proves only that the tree is broken somehow. -- MUST assert the **charter**, not just the id and the roster. A mailbox that bound its members while dropping its `MAILBOX.md` body has one of the four advertised conventions broken, and a check reading only the id and the members reports PASS on it. -- MUST compare **whole** stored records across the restart. `author` and `body` alone leave BR-15 weaker than the in-process claim above it, which already asserts `authorVerified`. -- MUST treat a bounded wait's **timeout as a failure**, never as a result. A helper that returns the last count it saw reports a run whose deliveries are still in flight as a settled transcript. -- MUST assert **complete sets**, not first elements. `mailboxIds` and `mailboxId` differ exactly when "the root produces one mailbox" stops being true, so reading the first element is reading the wrong thing. -- **And the same bar applies to the controls themselves.** A control that perturbs *two* things can be red because of the wrong one, and then it is decoration: it looks like coverage while proving nothing about the claim it names. The test to apply to every control is **"what would this report if the behaviour it guards were silently correct?"** — if the red survives, it is not measuring the claim. Where a control must bundle (the author gate needs post-backs suppressed to stay bounded), the second perturbation is held **identical across the green and red halves** so it cannot be the cause. +- MUST assert the **charter**, not just the id and the delegates. +- MUST compare **whole** stored records across the restart. The author and the body alone leave BR-15 weaker than the in-process claim above it. +- MUST treat a bounded wait's **timeout as a failure**, never as a result. +- MUST assert **complete sets**, not first elements. `coordinatorIds` and `coordinatorId` differ exactly when "the root produces one coordinator" stops being true. +- **And the same bar applies to the controls themselves.** The test to apply to every control is **"what would this report if the behaviour it guards were silently correct?"** — if the red survives, it is not measuring the claim. **Model:** n/a — handlers only. The gate is deliberately model-free: a model improvising around a missing document still reads as a pass (epic ER-7), so the contract gate grades what the plumbing carried. The model-backed claim is the sibling goal's. **Run:** `pnpm tsx goals/pentest-lab/a-post-reaches-both-declared-seats/run.mts` -**Controls:** `GOAL_CONTROL=` — `GOAL_CONTROL=list` prints them. Eleven go red; `reverse-order` is the one that must stay **green**, and deliberately so: it forces the two appends to land in the opposite order, so a check that had asserted an index would fail there and a correct one does not. +**Controls:** `GOAL_CONTROL=` — `GOAL_CONTROL=list` prints them. Eleven go red; `reverse-order` is the one that must stay **green**, and deliberately so: it forces the two answers to land in the opposite order, so a check that had asserted an index would fail there and a correct one does not. + +| Control | What it changes | Must fail at | +|---|---|---| +| `documents-swapped` | each seat redirected to its sibling's document, at the record | BR-3's document reads and leg c's document tokens | +| `reverse-order` | one seat's answer delayed | nothing: stays green | +| `fresh-store` | the rebuilt host pointed at an empty file | BR-15: 0 lines against 3 | +| `tamper-record` | one stored answer's `role` changed between the hosts, its author and body untouched | BR-15's whole-record comparison | +| `second-coordinator` | a second coordinator hired beside the tree's | BR-1's complete set, one clause | +| `drop-charter` | the coordinator hired with its `WORKER.md` body dropped | the charter, one clause | +| `answers-go-on` | the coordinator hired with `rounds: 1` | BR-8: an answer routed back out, five lines | +| `no-principal-org` | the host's `resolvePrincipal` left out | the first read, refused by org | +| `scribe-delegate` | `audit.scribe` added to the coordinator's delegates | BR-13: handed the post, given a session, its tokens in the conversation | +| `withhold-answer` | one answer withheld past a shortened bound | BR-19: the wait names the count it saw | +| `other-team-skill` | the audit team's `port-scan` handed to a pentest seat | BR-3's body token, not the name | +| `drop-shared-skill` | a shared skill taken from one seat | BR-3's set equality | ## Verdict log @@ -73,3 +97,6 @@ The one deliberate inversion: **the `scope-check` tool's clearance value is mint | 2026-10-01 | `f5f4d9342` | n/a | FAIL (expected) | `GOAL_CONTROL=tamper-principal`, repaired. Since posts became `mailbox-post` items (FIX-1585) the session's `transcript` holds no posted lines, so the control died at setup (*no authored line to tamper with*). It now blanks `principal` on the stored item and is red where it was designed to be: *after the restart the transcript differs: 3 line(s) against 3 before; changed line(s): …*, and *a restored line carries no principal*. | | 2026-10-01 | `f5f4d9342` | n/a | FAIL / PASS (expected) | The other ten controls re-run: `documents-swapped`, `fresh-store`, `second-mailbox`, `drop-charter`, `route-authored`, `scribe-member`, `withhold-answer`, `other-team-skill` and `drop-shared-skill` each red on their own clause, as in the 2026-09-17 rows; `reverse-order` green. | | 2026-10-01 | `10acd61ea` (review fixes: typed `tamperPrincipal`, one `act` call for the org-less inspect) | n/a | **PASS** | Gate re-run: all ten legs green, BR-16 refused at `runAction` as `OrgRequired`. All twelve controls re-run as in the rows above: eleven red on their own clause (`tamper-principal` on the whole-record comparison), `reverse-order` green. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | n/a | FAIL (old check) | **Rewritten for FIX-1792 P1; the check before the rewrite taken first.** The previous `run.mts` and fixture, run against the converted lab: `TypeError: Cannot read properties of undefined (reading 'length')` at leg a. The lab hires a coordinator and keeps no mailbox or notify log, so the old check cannot read it. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | n/a | **PASS** | All ten legs. One root produced 3 seats, 2 documents and the coordinator `pentest.findings`, its delegates read off the conversation and its charter token in the body the installation holds; each seat reported its exact expected union; one post was routed once, by everyone, to exactly `pentest.recon` and `pentest.triage`, both answers drawn under their seats with their own tokens and neither sibling's; three lines, no decision after the post's, and `audit.scribe` never routed to, given a session or mentioned; BR-16 refused `OrgRequired`; the SQLite rebuild read the same three message items whole. BR-9 skipped `pentest.triage` with *Worker "pentest.triage" runs on flow "probe-desk", which can't take a delegated post.* while `pentest.recon` answered. BR-10 refused at load: *createWorkerInstallation: standard worker "pentest.briefing" names "pentest.nobody" in `delegates:`, which isn't a standard worker.*; without `pentest.nobody` it opened and `pentest.recon` answered. BR-17 refused an `author` claim and wrote nothing; the same post without it was accepted. Both bad trees refused naming the worker; both twins hired 2 seats. | +| 2026-10-09 | same | n/a | FAIL / PASS (expected) | All twelve controls, each on its own run. `documents-swapped` red on both seats' document reads and both lines' document tokens. `fresh-store` red: *0 line(s) against 3 before*. `tamper-record` red on the whole-record comparison, naming the changed item, and on the reach check (*stored as a "system" message*). `second-coordinator` red on one clause: *coordinators ["pentest.briefing","pentest.findings"]*. `drop-charter` red on one clause: *charter: ""*. `answers-go-on` red on BR-8: five lines, one routing decision after the post's, each seat 2 lines. `no-principal-org` red at the first read: *Session … is bound to org `__fsd_default_org__` but request supplied org org_pentest_lab*. `scribe-delegate` red on BR-13: routed to, given a session, both its tokens in the conversation (and on leg a's delegate reads, since it is a delegate now). `withhold-answer` red with the count named: *saw 2 line(s), wanted 3*. `other-team-skill` red on the body token. `drop-shared-skill` red on set equality. `reverse-order` green. | diff --git a/goals/pentest-lab/a-post-reaches-both-declared-seats/run.mts b/goals/pentest-lab/a-post-reaches-both-declared-seats/run.mts index 26a4d2c1cb..8db8afd8f2 100644 --- a/goals/pentest-lab/a-post-reaches-both-declared-seats/run.mts +++ b/goals/pentest-lab/a-post-reaches-both-declared-seats/run.mts @@ -2,13 +2,13 @@ * Goal check — a post reaches both declared seats, and each answers as its own * folders described it. * - * **The gate.** Four file conventions — workers, skills, resources, mailboxes — - * have each shipped with a green suite of their own, and nothing has ever run - * them together. This does: one tree of Markdown, pointed at once, ends as a - * transcript in which two file-declared seats each answered a post using the - * document, skills and instructions their own folders gave them, with a third - * seat in another team silent — read back after the store is closed and the - * host rebuilt. + * **The gate.** Three file conventions — workers, skills, resources — have + * each shipped with a green suite of their own, and a coordinator is a worker + * file too. This runs them together: one tree of Markdown, pointed at once, + * ends as the operator's conversation with a coordinator in which two + * file-declared seats each answered a post using the document, skills and + * instructions their own folders gave them, with a third seat in another team + * silent — read back after the store is closed and the host rebuilt. * * Model-free on purpose. A model improvising around a missing document still * reads as a pass, so the contract gate puts a handler in the answering slot @@ -19,12 +19,12 @@ * Legs, and the rules each one closes: * * 0 the tree's held-out tokens live where the fixture says, and nowhere else - * a BR-1 BR-5 — the tree alone produces the roster, documents and mailbox + * a BR-1 BR-5 — the tree alone produces the roster, documents and coordinator * b BR-3 — each seat's exact skill union, read off the seat itself - * c BR-6 BR-7 BR-11 BR-12 BR-14 — one post, both members, three lines - * d BR-8 BR-13 — a seat's own line wakes nobody; the third seat is untouched + * c BR-6 BR-7 BR-11 BR-12 BR-14 — one post, both delegates, three lines + * d BR-8 BR-13 — an answer goes back out to nobody; the third seat is untouched * e BR-16 — an org-less request is refused at `runAction`, the door - * f BR-15 — closed, rebuilt, and the transcript reads the same + * f BR-15 — closed, rebuilt, and the conversation reads the same * g BR-9 BR-10 BR-17 — the second paths, each with its opposite state * h BR-18 — two bad trees refuse the whole hire; corrected, both hire * i BR-19 — a withheld answer fails loudly, naming the line count @@ -41,17 +41,16 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { handler } from "@flow-state-dev/core"; import { createSQLiteStores, sqliteStores } from "@flow-state-dev/store-sqlite"; -import { MAILBOX_POST_COMPONENT } from "@flow-state-dev/workforce"; -import { readMailboxesDirectory } from "@flow-state-dev/workforce/loader"; import { z } from "zod"; import { loadFixture, runGoal, silentLogger, stripIntentOverrides } from "../../lib/index.mts"; import { LAB_TREE, + loadTree, openLab, + type ConversationLine, type Lab, type OpenLabOptions, type SeatSkill, - type TranscriptLine, } from "../lab/host.mts"; import { SEAT_KIND, seatFactsSchema, type SeatFacts } from "../lab/workforce/flows/workers/probe.mts"; @@ -67,19 +66,19 @@ interface SeatFixture { interface Fixture { post: string; - mailbox: { id: string; members: string[]; charterToken: string }; + coordinator: { id: string; delegates: string[]; charterToken: string }; documents: Record; seats: Record; skillTokens: Record>; tokenHomes: Record; - members: string[]; + delegates: string[]; silentSeat: string; - nonMemberAuthor: string; + claimedAuthor: string; refusalTrees: Record< string, { dir: string; correctedDir: string; worker: string; names: string[]; otherWorker: string } >; - unknownMemberScenario: { dir: string; mailboxId: string; unknownMember: string }; + unknownDelegateScenario: { dir: string; coordinatorId: string; unknownDelegate: string }; wait: { timeoutMs: number; pollMs: number; shortTimeoutMs: number; withheldDelayMs: number }; } @@ -93,12 +92,12 @@ const CONTROLS = [ "documents-swapped", "reverse-order", "fresh-store", - "tamper-principal", - "second-mailbox", + "tamper-record", + "second-coordinator", "drop-charter", - "route-authored", + "answers-go-on", "no-principal-org", - "scribe-member", + "scribe-delegate", "withhold-answer", "other-team-skill", "drop-shared-skill", @@ -108,23 +107,35 @@ if (CONTROL === "list") { console.log(`controls: ${CONTROLS.join(", ")}`); process.exit(0); } +if (CONTROL !== "" && !(CONTROLS as readonly string[]).includes(CONTROL)) { + throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${CONTROLS.join(", ")}`); +} const LAB_ROOT = fileURLToPath(new URL("../lab", import.meta.url)); const REFUSAL_ROOT = join(LAB_ROOT, "refusal-trees"); const SCENARIO_ROOT = join(LAB_ROOT, "scenario-trees"); /** - * The scenario tree's mailbox, read once through the real mailbox loader. + * The scenario tree's coordinator, read once through the real worker loader. * * Two legs want it and one of them (`controlOptions`) is synchronous, so it is - * read here rather than twice inside the run: BR-10 opens it beside the tree's - * mailbox, and the `second-mailbox` control uses it as a stand-in for a loader - * that started returning an extra mailbox. + * read here rather than twice inside the run: BR-10 hires it beside the + * tree's workers, and the `second-coordinator` control hires it, with its + * unknown delegate taken off, as a stand-in for a loader that started + * returning an extra coordinator. */ -const scenarioRead = await readMailboxesDirectory( - join(SCENARIO_ROOT, fixture.unknownMemberScenario.dir) -); -const scenarioMailboxes = () => scenarioRead.mailboxes; +const scenarioRead = await loadTree(join(SCENARIO_ROOT, fixture.unknownDelegateScenario.dir)); +const scenarioCoordinator = () => { + const record = scenarioRead.workers.find((w) => w.id === fixture.unknownDelegateScenario.coordinatorId); + if (record === undefined) throw new Error(`the scenario tree declares no ${fixture.unknownDelegateScenario.coordinatorId}`); + return record; +}; +/** The scenario's coordinator with its unknown delegate taken off: a coordinator the installation accepts. */ +const scenarioCoordinatorKnown = () => { + const record = scenarioCoordinator(); + const delegates = (record.declared.delegates as string[]).filter((d) => d !== fixture.unknownDelegateScenario.unknownDelegate); + return { ...record, declared: { ...record.declared, delegates } }; +}; // --------------------------------------------------------------------------- // The answering block — the one thing the model-backed sibling replaces. @@ -137,7 +148,7 @@ const scenarioMailboxes = () => scenarioRead.mailboxes; const FIELD = " || "; /** How long this run's withheld seat sleeps, and which seat it is. */ -const withheld = CONTROL === "withhold-answer" ? fixture.members[0]! : undefined; +const withheld = CONTROL === "withhold-answer" ? fixture.delegates[0]! : undefined; /** * The bound leg (c) waits under. @@ -149,8 +160,8 @@ const withheld = CONTROL === "withhold-answer" ? fixture.members[0]! : undefined * a control which proves nothing. */ const mainWait = withheld === undefined ? wait.timeoutMs : wait.shortTimeoutMs; -/** Which seat this run delays, so the two appends land in the opposite order. */ -const delayed = CONTROL === "reverse-order" ? fixture.members[0]! : undefined; +/** Which seat this run delays, so the two answers land in the opposite order. */ +const delayed = CONTROL === "reverse-order" ? fixture.delegates[0]! : undefined; const gateAnswer = handler({ name: "gate-answer", @@ -158,7 +169,7 @@ const gateAnswer = handler({ outputSchema: z.string(), execute: async (facts: SeatFacts): Promise => { // The two timing perturbations. Neither changes what the line SAYS, which - // is the point: one forces the opposite append order, the other withholds + // is the point: one forces the opposite landing order, the other withholds // an answer past the wait's bound. if (facts.member === delayed) await sleep(300); if (facts.member === withheld) await sleep(wait.withheldDelayMs); @@ -186,67 +197,66 @@ const sleep = (ms: number): Promise => // names what it saw — BR-19. No goal here spins ad hoc. // --------------------------------------------------------------------------- -class ShortTranscriptError extends Error { +class ShortConversationError extends Error { readonly seen: number; readonly wanted: number; constructor(seen: number, wanted: number, timeoutMs: number) { super( - `the fan-out did not complete inside ${timeoutMs}ms: saw ${seen} line(s), wanted ${wanted}. ` + - `Delivery is best-effort, so a short transcript is a failure and never a pass.` + `the deliveries did not complete inside ${timeoutMs}ms: saw ${seen} line(s), wanted ${wanted}. ` + + `Delivery is best-effort, so a short conversation is a failure and never a pass.` ); - this.name = "ShortTranscriptError"; + this.name = "ShortConversationError"; this.seen = seen; this.wanted = wanted; } } /** - * Wait until the mailbox holds `wanted` lines and its fan-out has settled. + * Wait until the conversation holds `wanted` lines and every request on it, + * and on the delegate sessions it opened, has settled. * * Both conditions, not just the count: a run that reached the count while a * delivery was still in flight could still grow, and asserting on it would be - * asserting on a moment rather than on a settled transcript. + * asserting on a moment rather than on a settled conversation. */ -async function awaitTranscript( +async function awaitConversation( lab: Lab, wanted: number, timeoutMs = wait.timeoutMs, - mailboxId?: string -): Promise { + coordinatorId?: string +): Promise { const deadline = Date.now() + timeoutMs; - let seen: TranscriptLine[] = []; + let seen: ConversationLine[] = []; while (Date.now() < deadline) { - seen = (await lab.readMailbox(mailboxId)).transcript; - if (seen.length >= wanted && (await lab.fanOutSettled(mailboxId))) return seen; + seen = (await lab.readConversation(coordinatorId)).lines; + if (seen.length >= wanted && (await lab.settled(coordinatorId))) return seen; await sleep(wait.pollMs); } - throw new ShortTranscriptError(seen.length, wanted, timeoutMs); + throw new ShortConversationError(seen.length, wanted, timeoutMs); } -/** The fan-out was still running when the bound ran out. Never a length. */ +/** The deliveries were still running when the bound ran out. Never a length. */ class UnsettledError extends Error { constructor(seen: number, timeoutMs: number) { super( - `the transcript never settled inside ${timeoutMs}ms: it held ${seen} line(s) and the ` + - `fan-out was still in progress. A timeout is a failure, not a result.` + `the conversation never settled inside ${timeoutMs}ms: it held ${seen} line(s) and a ` + + `delivery was still in progress. A timeout is a failure, not a result.` ); this.name = "UnsettledError"; } } /** - * Wait for the transcript to STOP growing — the settled length, not a length. + * Wait for the conversation to STOP growing — the settled length, not a length. * - * BR-8's claim is that the transcript settles at three lines and stays there, - * and "it is three right now" cannot say that. This watches for a quiet period - * *after* the fan-out has reached a terminal status. + * BR-8's claim is that the conversation settles at three lines and stays + * there, and "it is three right now" cannot say that. This watches for a quiet + * period *after* every delivery has reached a terminal status. * * **It throws on the bound rather than returning what it last saw.** Returning * the last count would report a run whose deliveries are still in flight as a - * settled transcript — the caller cannot tell a quiet mailbox from a slow one, - * and "it settled at three" would be printed about a run that had not settled - * at all. That is the same defect this lab exists to prove against, so it is - * not one to leave in the lab's own helper. + * settled conversation — the caller cannot tell a quiet coordinator from a + * slow one. */ async function settledLength( lab: Lab, @@ -257,11 +267,11 @@ async function settledLength( let stableSince = Date.now(); const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { - const seen = (await lab.readMailbox()).transcript.length; + const seen = (await lab.readConversation()).lines.length; if (seen !== last) { last = seen; stableSince = Date.now(); - } else if (Date.now() - stableSince >= quietMs && (await lab.fanOutSettled())) { + } else if (Date.now() - stableSince >= quietMs && (await lab.settled())) { return seen; } await sleep(wait.pollMs); @@ -273,8 +283,8 @@ async function settledLength( // Grading helpers. Parsing, set comparison, and the "absent" half. // --------------------------------------------------------------------------- -const lineOf = (transcript: TranscriptLine[], author: string): TranscriptLine | undefined => - transcript.find((line) => line.author === author); +const lineBy = (lines: ConversationLine[], author: string): ConversationLine[] => + lines.filter((line) => line.author === author); const fieldOf = (body: string, prefix: string): string => { const part = body.split(FIELD).find((chunk) => chunk.startsWith(prefix)); @@ -299,6 +309,13 @@ function filesUnder(root: string, prefix = ""): Array<[string, string]> { const messageOf = (error: unknown): string => error instanceof Error ? error.message : String(error); +/** The delegates a set of routing records delivered to, and the ones they skipped. */ +function routed(lab: { routes: Awaited>["routes"] }) { + const of = (outcome: string) => + lab.routes.flatMap((record) => record.delegates.filter((d) => d.outcome === outcome).map((d) => d.worker)); + return { delivered: of("delivered"), skipped: of("skipped"), failed: of("failed") }; +} + /** Open a lab over throwaway SQLite stores, with this run's control applied. */ async function open(over: Partial & { dbFile: string }): Promise { return await openLab({ @@ -312,7 +329,7 @@ async function open(over: Partial & { dbFile: string }): Promise /** The host options this run's control asks for. Empty for the real gate. */ function controlOptions(): Partial { - const [first, second] = fixture.members as [string, string]; + const [first, second] = fixture.delegates as [string, string]; switch (CONTROL) { case "documents-swapped": return { @@ -321,28 +338,24 @@ function controlOptions(): Partial { [second]: fixture.seats[first]!.document, }, }; - case "route-authored": - // Deliberately EMPTY here. This control belongs to one leg — the author - // gate's — and applying it globally is what made it decoration: it - // carried `suppressPostBack`, which on its own emptied the transcript and - // the authored log, so the red survived with authored routing turned off. - // The leg reads `CONTROL` itself and holds the suppression constant - // across both halves. - return {}; + case "answers-go-on": + // The coordinator hired with `rounds: 1`: each answer goes back out to + // the other delegate once, instead of stopping where it landed. + return { coordinatorRounds: 1 }; case "no-principal-org": return { omitPrincipal: true }; - case "scribe-member": - return { alsoMember: [fixture.silentSeat] }; - case "second-mailbox": - // A second mailbox bound alongside the tree's, standing in for a loader - // that started returning one. `pentest.findings` stays FIRST, so - // `lab.mailboxId` and every later operation are unaffected — which is - // exactly the shape that leaves a first-element check green. - return { extraMailboxes: scenarioMailboxes() }; + case "scribe-delegate": + return { alsoDelegate: [fixture.silentSeat] }; + case "second-coordinator": + // A second coordinator hired alongside the tree's, standing in for a + // loader that started returning one. `pentest.findings` stays the tree's + // own, so every later operation is unaffected — which is exactly the + // shape that leaves a first-element check green. + return { extraWorkers: [scenarioCoordinatorKnown()] }; case "drop-charter": - // The roster still binds and the mailbox still opens; only the - // `MAILBOX.md` body is gone. Every other leg stays green, which is what - // makes this the charter assertion's own control. + // The installation still builds and the post still routes; only the + // coordinator's `WORKER.md` body is gone. Every other leg stays green, + // which is what makes this the charter clause's own control. return { dropCharter: true }; case "other-team-skill": // The OTHER team's same-named `port-scan`, handed to a pentest seat. Set @@ -371,40 +384,33 @@ function controlOptions(): Partial { } /** - * Blank `principal` on one stored transcript line, between the two hosts. + * Change one field of one stored answer, between the two hosts: its `role`, + * from `assistant` to `system`. * - * Writes to the closed SQLite file directly rather than through a host, because - * the point is a restart that serves something subtly different from what was - * written — which no public route will do on request. `author` and `body` are - * left exactly as they were, so a comparison covering only those two reads this - * as unchanged. + * Writes to the closed SQLite file directly rather than through a host, + * because the point is a restart that serves something subtly different from + * what was written — which no public route will do on request. The author and + * the body are left exactly as they were, so a comparison covering only those + * two reads this as unchanged. * - * A posted line is a `mailbox-post` item on the request that posted it, not an - * entry in the mailbox session's state, so the edit goes to that stored item. - * The session's own `transcript` holds only lines kept before posts became - * items, and is empty on a mailbox opened today. + * An answer is a `message` item on the coordinator's request that landed it, + * so the edit goes to that stored item. */ -async function tamperPrincipal(dbFile: string, mailboxId: string): Promise { +async function tamperRecord(dbFile: string, author: string): Promise { const stores = createSQLiteStores({ filename: dbFile }); try { - // The first authored post found, which is one member's post-back. One line - // is all the control needs: the comparison names whichever line changed. - const requests = await stores.request.list({ sessionId: mailboxId, withItems: true }); - for (const request of requests) { + for (const request of await stores.request.list({ withItems: true } as never)) { const item = request.items?.find( - (entry) => - entry.type === "component" && - entry.component === MAILBOX_POST_COMPONENT && - entry.data.author !== undefined, + (entry) => entry.type === "message" && (entry as { agentName?: string }).agentName === author, ); - if (item?.type !== "component") continue; - item.data.principal = ""; + if (item?.type !== "message") continue; + (item as { role: string }).role = "system"; stores.request.persistItems(request.id, [item]); // The write is queued; this settles it before `close()` below. await stores.request.flushItems(request.id); return; } - throw new Error(`no authored line on "${mailboxId}" to tamper with`); + throw new Error(`no answer by ${author} stored to tamper with`); } finally { // Closed before the next host opens the same file — awaited by the caller, // so the two never hold it at once. @@ -429,7 +435,7 @@ await runGoal(async () => { const dir = mkdtempSync(join(tmpdir(), "fsd-pentest-lab-")); const dbFile = join(dir, "gate.db"); - const [firstMember, secondMember] = fixture.members as [string, string]; + const [firstDelegate, secondDelegate] = fixture.delegates as [string, string]; // ---- (0) the held-out tokens live where the fixture says, and only there -- // @@ -461,8 +467,8 @@ await runGoal(async () => { } const lab = await open({ dbFile }); - /** What the transcript held before the store was closed — leg (f) compares to this. */ - let beforeRestart: TranscriptLine[] = []; + /** What the conversation held before the store was closed — leg (f) compares to this. */ + let beforeRestart: ConversationLine[] = []; try { // ---- (a) BR-1, BR-5 — the tree alone produced all of it ---------------- @@ -473,16 +479,10 @@ await runGoal(async () => { } // Every seat is on the LAB's kind, asserted before anything downstream - // reads a document or a skill off one. - // - // A record whose `flow:` line went missing hires into the built-in `agent` - // kind instead, which declares neither `document` nor this lab's tool - // fence — and a seat that carries no document resource passes an - // isolation check trivially, because "it cannot read its sibling's" is - // true of a seat that can read nothing. Today that particular slip - // refuses at the mint (the built-in kind's closed schema rejects - // `document:`), but relying on a neighbouring refusal to catch it is - // exactly the shape this check exists to stop depending on. + // reads a document or a skill off one. A record whose `flow:` line went + // missing hires into the built-in `agent` kind instead, which declares + // neither `document` nor this lab's tool fence — and a seat that carries + // no document resource passes an isolation check trivially. for (const [seatId, seat] of Object.entries(lab.seats)) { if ((seat as { kind?: string }).kind !== SEAT_KIND) { note(`${seatId} was hired into kind "${String((seat as { kind?: string }).kind)}", not "${SEAT_KIND}"`); @@ -494,60 +494,60 @@ await runGoal(async () => { } // The COMPLETE set, not its first element. // - // BR-1's claim is that the root produces *one* mailbox, and `lab.mailboxId` - // reads `treeMailboxes[0]` — those two differ exactly when the claim breaks. - // A loader that started returning a second mailbox while keeping - // `pentest.findings` first would leave every later operation pointed at the - // right session and this leg green, with the claim violated. - if (!sameSet(lab.mailboxIds, [fixture.mailbox.id])) { + // BR-1's claim is that the root produces *one* coordinator, and + // `lab.coordinatorId` reads the tree's first — those two differ exactly + // when the claim breaks. A loader that started returning a second one + // while keeping `pentest.findings` would leave every later operation + // pointed at the right conversation and this leg green, with the claim + // violated. + if (!sameSet(lab.coordinatorIds, [fixture.coordinator.id])) { note( - `the root produced mailboxes ${JSON.stringify(lab.mailboxIds)}, ` + - `wanted exactly ${JSON.stringify([fixture.mailbox.id])}` + `the root produced coordinators ${JSON.stringify(lab.coordinatorIds)}, ` + + `wanted exactly ${JSON.stringify([fixture.coordinator.id])}` ); } - if (lab.mailboxId !== fixture.mailbox.id) { - note(`the mailbox opened at "${lab.mailboxId}", wanted "${fixture.mailbox.id}"`); + if (lab.coordinatorId !== fixture.coordinator.id) { + note(`the coordinator is "${lab.coordinatorId}", wanted "${fixture.coordinator.id}"`); } - const opened = await lab.readMailbox(); - if (!sameSet(opened.members, fixture.mailbox.members)) { - note(`the mailbox's members are ${JSON.stringify(opened.members)}`); + // Its delegates, through the coordinator's own `listDelegates` on the + // operator's conversation: the defaults the conversation starts from. + const opened = await lab.readConversation(); + if (!sameSet(opened.delegates, fixture.coordinator.delegates)) { + note(`the conversation's delegates are ${JSON.stringify(opened.delegates)}`); } - // The charter — the fourth convention, and the half BR-5 claims that the - // id and the roster cannot speak for. - // - // Read off the mailbox's stored session state rather than its `read` - // action, which projects `{ id, description, members, transcript }` and - // deliberately does not surface `instructions`. Without this, the whole - // manifest body could stop reaching the opened session — `stateFor`, - // `openMailboxes` or the reader dropping it — while the roster still bound, - // and this goal would still report PASS with one of its four advertised - // conventions broken. - const bound = await lab.mailboxState(); - if (!bound.instructions.includes(fixture.mailbox.charterToken)) { + // The charter — the coordinator's `WORKER.md` body, which is its + // instructions. A coordinator that routes to everyone runs no turn of + // its own, so nothing reads the body at run time; the deepest point it + // reaches is the worker the installation hands the `coordinator` flow, + // which is where it is read. Without this, the body could stop reaching + // the coordinator while its delegates still bound. + const worker = lab.coordinatorWorker(); + if (!worker.instructions.includes(fixture.coordinator.charterToken)) { note( - `the opened mailbox's charter does not carry ${fixture.mailbox.charterToken} — ` + - `its MAILBOX.md body did not reach the session (charter: ` + - `${JSON.stringify(bound.instructions.slice(0, 80))})` + `the coordinator's charter does not carry ${fixture.coordinator.charterToken} — ` + + `its WORKER.md body did not reach the installation (charter: ` + + `${JSON.stringify(worker.instructions.slice(0, 80))})` ); } - // Members are asserted here too, off the same read: the charter half must - // not be able to pass on a session that bound nothing at all. - if (!sameSet(bound.members, fixture.mailbox.members)) { - note(`the opened mailbox's stored members are ${JSON.stringify(bound.members)}`); + // Delegates are asserted here too, off the same read: the charter half + // must not be able to pass on a worker that bound nothing at all. + if (!sameSet(worker.delegates, fixture.coordinator.delegates)) { + note(`the coordinator worker's declared delegates are ${JSON.stringify(worker.delegates)}`); } evidence.push( - `one root produced ${seatIds.length} seats, ${docRefs.length} documents and the mailbox ` + - `"${lab.mailboxId}" carrying its two declared members and its MAILBOX.md body as its charter` + `one root produced ${seatIds.length} seats, ${docRefs.length} documents and the coordinator ` + + `"${lab.coordinatorId}" carrying its two declared delegates, read off the conversation, and its ` + + `WORKER.md body as its charter` ); } // ---- (b) BR-3 — each seat's exact union, read OFF THE SEAT ------------- // // Through the seat's own config inside a running block, never off the - // hire's return value, and never off the transcript — which by BR-13 can + // hire's return value, and never off the conversation — which by BR-13 can // never carry the silent seat at all. { for (const [seatId, expected] of Object.entries(fixture.seats)) { @@ -612,88 +612,95 @@ await runGoal(async () => { } // ---- (c) BR-6, BR-7, BR-11, BR-12, BR-14 — one post, three lines ------- - let transcript: TranscriptLine[] = []; + let lines: ConversationLine[] = []; { const posted = await lab.post(fixture.post); if (posted.error !== undefined) note(`the operator's post was refused: ${posted.error}`); try { - transcript = await awaitTranscript(lab, 3, mainWait); + lines = await awaitConversation(lab, 3, mainWait); } catch (error) { note(messageOf(error)); - transcript = (await lab.readMailbox()).transcript; + lines = (await lab.readConversation()).lines; } - if (!sameSet(lab.notifyLog.addressed, fixture.members)) { - note( - `the fan-out addressed ${JSON.stringify(lab.notifyLog.addressed)}, ` + - `wanted exactly the two declared members` - ); + // Who the coordinator handed the post to, off its own routing record. + const read = await lab.readConversation(); + const firstRound = read.routes.filter((record) => record.round === 0); + const { delivered } = routed({ routes: firstRound }); + if (firstRound.length !== 1 || firstRound[0]!.by !== "everyone") { + note(`the post was routed ${firstRound.length} time(s), by ${JSON.stringify(firstRound.map((r) => r.by))} (want once, by everyone)`); + } + if (!sameSet(delivered, fixture.delegates)) { + note(`the coordinator delivered the post to ${JSON.stringify(delivered)}, wanted exactly the two declared delegates`); } - // Presence and attribution, NEVER index — two concurrent appends have no + // Presence and attribution, NEVER index — two concurrent answers have no // order, and asserting one is how a correct implementation fails. - for (const member of fixture.members) { - const line = lineOf(transcript, member); - if (line === undefined) { - note(`${member} put no line in the transcript`); + for (const delegate of fixture.delegates) { + const own = lineBy(lines, delegate); + if (own.length !== 1) { + note(`${delegate} put ${own.length} line(s) in the conversation, wanted 1`); continue; } - if (line.authorVerified !== false) { - note(`${member}'s line stored authorVerified=${String(line.authorVerified)}, wanted false`); + const line = own[0]!; + // The coordinator draws an answer under the delegate its delivery + // named; the answer itself carries no author it could claim. + if (line.role !== "assistant") { + note(`${delegate}'s line is stored as a "${line.role}" message, wanted an assistant answer`); } - const expected = fixture.seats[member]!; - const sibling = member === firstMember ? secondMember : firstMember; + const expected = fixture.seats[delegate]!; + const sibling = delegate === firstDelegate ? secondDelegate : firstDelegate; const siblingFixture = fixture.seats[sibling]!; if (!line.body.includes(expected.instructionsToken)) { - note(`${member}'s line does not carry its own instructions token ${expected.instructionsToken}`); + note(`${delegate}'s line does not carry its own instructions token ${expected.instructionsToken}`); } const ownDocToken = fixture.documents[expected.document]!; if (!line.body.includes(ownDocToken)) { - note(`${member}'s line does not carry its own document token ${ownDocToken}`); + note(`${delegate}'s line does not carry its own document token ${ownDocToken}`); } // The absent half, asserted ONLY of what is genuinely seat-exclusive: // the sibling's instructions and the sibling's document. Not of the // skills both pentest seats legitimately share. if (line.body.includes(siblingFixture.instructionsToken)) { - note(`${member}'s line carries ${sibling}'s instructions token`); + note(`${delegate}'s line carries ${sibling}'s instructions token`); } const siblingDocToken = fixture.documents[siblingFixture.document]!; if (siblingDocToken !== ownDocToken && line.body.includes(siblingDocToken)) { - note(`${member}'s line carries ${sibling}'s document token ${siblingDocToken}`); + note(`${delegate}'s line carries ${sibling}'s document token ${siblingDocToken}`); } const skills = fieldOf(line.body, "skills=").split(",").filter((s) => s.length > 0); if (!sameSet(skills, expected.skills)) { note( - `${member}'s line reports skills ${JSON.stringify(skills)}, ` + + `${delegate}'s line reports skills ${JSON.stringify(skills)}, ` + `wanted exactly ${JSON.stringify(expected.skills)}` ); } // The one genuinely seat-exclusive skill: a worker-level folder. const exclusive = expected.skills.filter((s) => !siblingFixture.skills.includes(s)); for (const name of siblingFixture.skills.filter((s) => !expected.skills.includes(s))) { - if (skills.includes(name)) note(`${member}'s line holds ${sibling}'s worker-level "${name}"`); + if (skills.includes(name)) note(`${delegate}'s line holds ${sibling}'s worker-level "${name}"`); } - if (member === firstMember && exclusive.length === 0) { - note(`the fixture gives ${member} no seat-exclusive skill, so the absent half proves nothing`); + if (delegate === firstDelegate && exclusive.length === 0) { + note(`the fixture gives ${delegate} no seat-exclusive skill, so the absent half proves nothing`); } } - const operator = transcript.filter((line) => line.author === undefined); - if (operator.length !== 1 || operator[0]!.body !== fixture.post) { - note(`the operator's line is not in the transcript exactly once`); + const operator = lines.filter((line) => line.role === "user"); + if (operator.length !== 1 || operator[0]!.body !== fixture.post || operator[0]!.author !== null) { + note(`the operator's post is not in the conversation exactly once, under no delegate's name`); } evidence.push( - `one post woke exactly the two declared members; all three lines are present, each member's ` + - `carrying its own instructions, document and exact skill union and neither sibling's` + `one post reached exactly the two declared delegates; all three lines are present, each ` + + `delegate's carrying its own instructions, document and exact skill union and neither sibling's` ); } - // ---- (d) BR-8, BR-13 — the cycle break, and the seat nobody woke ------- + // ---- (d) BR-8, BR-13 — the cycle break, and the seat nobody reached ---- { // Under the withheld-answer control this bound is the short one, and the // withheld seat is still answering when it runs out — so this is also @@ -701,94 +708,43 @@ await runGoal(async () => { // a length. try { const settled = await settledLength(lab, 400, mainWait); - if (settled !== 3) note(`the transcript settled at ${settled} lines, wanted 3`); + if (settled !== 3) note(`the conversation settled at ${settled} lines, wanted 3`); } catch (error) { note(messageOf(error)); } - // Both members' own post-backs reached the router and routed to nobody. - if (!sameSet([...new Set(lab.notifyLog.authored)], fixture.members)) { + // Both answers landed and went no further: one routing decision, the + // operator's post's. An answer routed back out is a second one. + const read = await lab.readConversation(); + const later = read.routes.filter((record) => record.round > 0); + if (later.length > 0) { note( - `the router saw authored lines from ${JSON.stringify([...new Set(lab.notifyLog.authored)])}, ` + - `wanted both members' own post-backs` + `an answer was routed back out: ${later.length} routing decision(s) after the post's, delivering to ` + + `${JSON.stringify(routed({ routes: later }).delivered)} — an answer must wake nobody` ); } - // BR-13, both halves. The dispatch record is the half that matters: a - // stray delivery whose seat produces no post is otherwise indistinguish- - // able from no delivery at all. - if (lab.notifyLog.addressed.includes(fixture.silentSeat)) { - note(`the fan-out dispatched to ${fixture.silentSeat}`); + // BR-13, both halves. The routing record and the delegate sessions are + // the half that matters: a stray delivery whose seat produces no answer + // is otherwise indistinguishable from no delivery at all. + const everyRoute = routed(read); + if ([...everyRoute.delivered, ...everyRoute.skipped, ...everyRoute.failed].includes(fixture.silentSeat)) { + note(`the coordinator routed a post to ${fixture.silentSeat}`); + } + const sessions = await lab.delegateSessions(); + if (!sameSet([...new Set(sessions)], fixture.delegates) || sessions.length !== fixture.delegates.length) { + note(`the conversation opened delegate sessions for ${JSON.stringify(sessions)}, wanted one each for the two delegates`); } const silentToken = fixture.seats[fixture.silentSeat]!.instructionsToken; const silentSkillToken = fixture.skillTokens[fixture.silentSeat]!["port-scan"]!; - const whole = transcript.map((line) => line.body).join("\n"); - if (whole.includes(silentToken)) note(`${fixture.silentSeat}'s token is in the transcript`); + const whole = read.lines.map((line) => line.body).join("\n"); + if (whole.includes(silentToken)) note(`${fixture.silentSeat}'s token is in the conversation`); if (whole.includes(silentSkillToken)) { - note(`${fixture.silentSeat}'s team-level skill token is in the transcript`); - } - evidence.push( - `the transcript settled at three lines and stayed there — each member's own post-back reached ` + - `the router and woke nobody — and ${fixture.silentSeat} was neither dispatched to nor mentioned` - ); - } - - // ---- (d2) BR-8 — the author gate, on a line posted AS a member --------- - // - // Leg (d) above observes the gate on the main path, where the seats' own - // post-backs are the authored lines. It cannot carry the gate's control, - // though: turning authored routing on there makes the run 1 → 2 → 4 → 8 - // detached dispatches, and the only way to bound that is to suppress the - // post-backs — which removes the authored lines altogether and leaves the - // control red whether or not the gate is doing anything. - // - // So the claim gets its own lab, where **post-backs are suppressed in both - // halves** and the operator posts one already-authored line by hand. The - // suppression is therefore constant and cannot be the reason for the red; - // the only thing that varies is the gate. - { - const gated = await open({ - dbFile: join(dir, "br8.db"), - suppressPostBack: true, - ...(CONTROL === "route-authored" ? { routeAuthoredLines: true } : {}), - }); - try { - const posted = await gated.post(fixture.post, firstMember); - if (posted.error !== undefined) { - note(`BR-8: a declared member's own line was refused — ${posted.error}`); - } - // The line landed, so the router certainly saw it — without this the - // clauses below are true of a post that never happened. - const seen = await awaitTranscript(gated, 1).catch( - async () => (await gated.readMailbox()).transcript - ); - if (lineOf(seen, firstMember) === undefined) { - note(`BR-8: the authored line is not in the transcript, so the router never saw it`); - } - await settledLength(gated, 400).catch(() => undefined); - - // The gate's two halves: the router RECORDED the authored line, and - // dispatched to nobody on account of it. - if (!sameSet([...new Set(gated.notifyLog.authored)], fixture.members)) { - note( - `BR-8: the router recorded authored lines from ` + - `${JSON.stringify([...new Set(gated.notifyLog.authored)])}, wanted both members — ` + - `an authored line reached it once per declared member` - ); - } - if (gated.notifyLog.addressed.length > 0) { - note( - `BR-8: an already-authored line was dispatched to ` + - `${JSON.stringify(gated.notifyLog.addressed)} — a seat's own line must wake nobody` - ); - } - } finally { - await gated.dispose(); + note(`${fixture.silentSeat}'s team-level skill token is in the conversation`); } evidence.push( - `an already-authored line reaches the router once per declared member and is dispatched to ` + - `nobody — observed on the dispatch, with the post-backs suppressed identically in the ` + - `control, so the suppression cannot be what makes it red` + `the conversation settled at three lines and stayed there — each answer landed and was routed to ` + + `nobody — and ${fixture.silentSeat} was neither routed to, given a session, nor mentioned` ); } @@ -810,7 +766,7 @@ await runGoal(async () => { `one lands — so the refusal is the org and not the door` ); } - beforeRestart = (await lab.readMailbox()).transcript; + beforeRestart = (await lab.readConversation()).lines; } finally { // `dispose()` closes the SQLite adapter, which is what makes the next host // a genuine reopen of the file rather than a second handle on a live one. @@ -825,43 +781,37 @@ await runGoal(async () => { // different file → none. const secondFile = CONTROL === "fresh-store" ? join(dir, "empty.db") : dbFile; - // The control for the STRENGTHENED half. `fresh-store` proves the lines come - // off a file; it cannot prove this comparison covers more than `author` and - // `body`, because it empties everything at once. This one edits the stored - // file between the two hosts and blanks `principal` on one line, leaving - // author and body exactly as they were — invisible to the old comparison, - // red to this one. - if (CONTROL === "tamper-principal") await tamperPrincipal(dbFile, fixture.mailbox.id); + // The control for the whole-record half. `fresh-store` proves the lines + // come off a file; it cannot prove this comparison covers more than the + // author and the body, because it empties everything at once. This one + // edits the stored file between the two hosts and changes one answer's + // role, leaving its author and body exactly as they were — invisible to an + // author-and-body comparison, red to this one. + if (CONTROL === "tamper-record") await tamperRecord(dbFile, firstDelegate); const rebuilt = await open({ dbFile: secondFile }); try { - const after = (await rebuilt.readMailbox()).transcript; - - // **The WHOLE record, every field, normalised only for order.** - // - // Comparing `author` and `body` alone was the weaker claim it looked - // like: a restart that lost or changed `principal`, `authorVerified`, a - // line's id or its timestamp would produce the same string and still be - // called "line for line". The asymmetry was the tell — leg (c) asserts - // `authorVerified` *before* the restart, and the restart is the part BR-15 - // is actually about, so excluding it afterwards left the persistence claim - // weaker than the in-process one. - // - // Sorted by id rather than compared positionally, because BR-14 forbids - // asserting an order and a restart is no excuse to start. - const shape = (lines: TranscriptLine[]): string => - lines - .map((line) => JSON.stringify(line, Object.keys(line).sort())) + const after = (await rebuilt.readConversation()).lines; + + // **The WHOLE record, every field, normalised only for order.** Comparing + // the author and the body alone would let a restart that lost or changed + // a line's id, role, timestamp or any other stored field produce the + // same string and still be called "line for line". Sorted by id rather + // than compared positionally, because BR-14 forbids asserting an order + // and a restart is no excuse to start. + const shape = (rows: ConversationLine[]): string => + rows + .map((line) => JSON.stringify(line.item, Object.keys(line.item).sort())) .sort() .join("\n"); if (shape(after) !== shape(beforeRestart)) { - const before = new Map(beforeRestart.map((line) => [line.id, line])); + const before = new Map(beforeRestart.map((line) => [line.id, JSON.stringify(line.item, Object.keys(line.item).sort())])); const changed = after - .filter((line) => JSON.stringify(before.get(line.id) ?? null) !== JSON.stringify(line)) + .filter((line) => before.get(line.id) !== JSON.stringify(line.item, Object.keys(line.item).sort())) .map((line) => line.id); note( - `after the restart the transcript differs: ${after.length} line(s) against ` + + `after the restart the conversation differs: ${after.length} line(s) against ` + `${beforeRestart.length} before` + (changed.length > 0 ? `; changed line(s): ${changed.join(", ")}` : "") + (after.length === beforeRestart.length && changed.length === 0 @@ -870,24 +820,23 @@ await runGoal(async () => { ); } - // The comparison must actually REACH the fields it now covers. Without - // this, a transcript whose lines somehow carried none of them would - // compare equal to itself and the strengthening would be invisible. + // The comparison must actually REACH the fields it covers. Without this, + // a conversation whose lines somehow carried none of them would compare + // equal to itself and the whole-record comparison would be invisible. for (const line of after) { if (typeof line.id !== "string" || line.id.length === 0) { note(`a restored line carries no id, so the comparison has nothing to key on`); } - if (typeof line.principal !== "string" || line.principal.length === 0) { - note(`a restored line carries no principal`); + if (line.role !== "user" && line.role !== "assistant") { + note(`a restored line is stored as a "${line.role}" message, wanted user or assistant`); } - if (line.authorVerified !== false) { - note(`a restored line has authorVerified=${String(line.authorVerified)}, wanted false`); + if (line.role === "assistant" && (line.author === null || !fixture.delegates.includes(line.author))) { + note(`a restored answer is drawn under ${JSON.stringify(line.author)}, not a delegate`); } - if (typeof line.at !== "number") note(`a restored line carries no timestamp`); } evidence.push( - "the store was closed and a fresh host built over the same file; the transcript reads the " + - "same, line for line" + "the store was closed and a fresh host built over the same file; the conversation reads the " + + "same, line for line, every stored field" ); } finally { await rebuilt.dispose(); @@ -896,119 +845,108 @@ await runGoal(async () => { // ---- (g) BR-9, BR-10, BR-17 — the second paths, each with its opposite --- { - // BR-9: a declared member with no address is recorded and skipped, and the - // other still runs. Paired with its opposite state below, because a router - // that does nothing at all satisfies the skip half on its own. - const withoutAddress = await open({ + // BR-9: a declared delegate whose flow takes no posts is skipped, with + // why, on the routing record, and the other still answers. Paired with its + // opposite state — the same delegate on the lab's kind lands (leg c) — + // because a coordinator that does nothing at all satisfies the skip half + // on its own. + const withoutPosts = await open({ dbFile: join(dir, "br9.db"), - omitAddressFor: [secondMember], + noPostFor: [secondDelegate], }); try { - await withoutAddress.post(fixture.post); - const seen = await awaitTranscript(withoutAddress, 2).catch( - async () => (await withoutAddress.readMailbox()).transcript + await withoutPosts.post(fixture.post); + const seen = await awaitConversation(withoutPosts, 2).catch( + async () => (await withoutPosts.readConversation()).lines ); - if (!withoutAddress.notifyLog.skipped.includes(secondMember)) { - note(`BR-9: ${secondMember} was not recorded as skipped`); + const read = await withoutPosts.readConversation(); + const skip = read.routes.flatMap((record) => record.delegates).find((d) => d.worker === secondDelegate); + if (skip?.outcome !== "skipped" || (skip.reason ?? "").length === 0) { + note(`BR-9: ${secondDelegate} was not recorded as skipped with a reason (${JSON.stringify(skip ?? null)})`); } - if (lineOf(seen, firstMember) === undefined) { - note(`BR-9: ${firstMember} did not still run while its co-member had no address`); + if (lineBy(seen, firstDelegate).length !== 1) { + note(`BR-9: ${firstDelegate} did not still answer while its co-delegate could take no post`); } - if (lineOf(seen, secondMember) !== undefined) { - note(`BR-9: ${secondMember} answered although it had no address`); + if (lineBy(seen, secondDelegate).length > 0) { + note(`BR-9: ${secondDelegate} answered although its flow takes no posts`); } - const members = (await withoutAddress.readMailbox()).members; - if (!sameSet(members, fixture.mailbox.members)) { - note(`BR-9: a skipped delivery changed the mailbox's membership to ${JSON.stringify(members)}`); + if (!sameSet(read.delegates, fixture.coordinator.delegates)) { + note(`BR-9: a skipped delivery changed the conversation's delegates to ${JSON.stringify(read.delegates)}`); + } + if (skip?.outcome === "skipped") { + evidence.push( + `a declared delegate whose flow takes no posts is skipped with why ("${skip.reason}") while the ` + + `other still answers — and the same delegate, on the lab's kind, lands (leg c)` + ); } } finally { - await withoutAddress.dispose(); + await withoutPosts.dispose(); } - // The opposite state is the real gate above: the same member, addressed, - // does land. Named here so the pair reads as one check. - evidence.push( - `a declared member with no address is recorded and skipped while the other still answers — and ` + - `the same member, addressed, lands (leg c)` - ); - // BR-10 + BR-17, on one host. The extra mailbox is read through the real - // mailbox loader, so it is a MAILBOX.md rather than a record this file - // wrote — which is what BR-10 is about. - if (scenarioRead.errors.length > 0) { - note( - `BR-10: the scenario tree did not load — ` + - `${scenarioRead.errors.map((e) => e.path).join(", ")}` - ); + // BR-10: a coordinator naming a delegate no file declares is refused at + // load, naming it. The coordinator is read through the real worker + // loader, so it is a WORKER.md rather than a record this file wrote. + let refusal = ""; + try { + const refusedLab = await open({ dbFile: join(dir, "br10.db"), extraWorkers: [scenarioCoordinator()] }); + note(`BR-10: a coordinator naming ${fixture.unknownDelegateScenario.unknownDelegate} opened instead of being refused`); + await refusedLab.dispose(); + } catch (error) { + refusal = messageOf(error); + } + if (refusal !== "") { + for (const name of [fixture.unknownDelegateScenario.coordinatorId, fixture.unknownDelegateScenario.unknownDelegate]) { + if (!refusal.includes(name)) note(`BR-10: the refusal does not name "${name}": ${refusal}`); + } } - const scenario = await open({ - dbFile: join(dir, "br10.db"), - extraMailboxes: scenarioMailboxes(), - }).catch((error) => error as Error); - if (scenario instanceof Error) { - note(`BR-10: the unknown-member scenario could not be opened — ${scenario.message}`); + // Its opposite state, on the same record with the unknown delegate taken + // off: it opens, and a post to it reaches its one known delegate. Without + // this, a host that refuses every second coordinator passes the half above. + const known = await open({ dbFile: join(dir, "br10-known.db"), extraWorkers: [scenarioCoordinatorKnown()] }).catch( + (error) => error as Error + ); + if (known instanceof Error) { + note(`BR-10: the scenario coordinator, its unknown delegate taken off, was refused too — ${known.message}`); } else { try { - // BR-10 — a MAILBOX.md naming a member that is no hired seat at all. - // Distinct from BR-9's hired-but-unmapped seat: this id is in no - // `workers/` folder anywhere, so the question is whether OPENING - // survives a roster entry with no worker behind it. - const extraId = fixture.unknownMemberScenario.mailboxId; - if (!scenario.mailboxIds.includes(extraId)) { - note(`BR-10: the mailbox declaring an unknown member did not open`); - } else { - const opened = await scenario.readMailbox(extraId); - if (!opened.members.includes(fixture.unknownMemberScenario.unknownMember)) { - note(`BR-10: the unknown member was dropped from the declared roster`); - } - await scenario.post(fixture.post, undefined, extraId); - // The POSITIVE half first, and it is not decoration: everything below - // is about what did NOT happen, and every one of those is trivially - // true on a mailbox whose fan-out never ran at all. A fixture that - // fails to wire the thing under test produces a pass, not an error. - let landed: TranscriptLine[] = []; - try { - landed = await awaitTranscript(scenario, 2, wait.timeoutMs, extraId); - } catch (error) { - note(`BR-10: the mailbox's own hired member never answered — ${messageOf(error)}`); - } - if (lineOf(landed, firstMember) === undefined) { - note(`BR-10: ${firstMember} is a declared member of this mailbox and did not answer`); - } - if (!scenario.notifyLog.addressed.includes(firstMember)) { - note(`BR-10: the fan-out on this mailbox dispatched to nobody, so it proves nothing`); - } - - if (scenario.notifyLog.addressed.includes(fixture.unknownMemberScenario.unknownMember)) { - note(`BR-10: the fan-out dispatched to a member that is no hired seat`); - } - if (!scenario.notifyLog.skipped.includes(fixture.unknownMemberScenario.unknownMember)) { - note(`BR-10: the unknown member was not recorded and skipped`); - } + const scenarioId = fixture.unknownDelegateScenario.coordinatorId; + await known.post(fixture.post, undefined, scenarioId); + let landed: ConversationLine[] = []; + try { + landed = await awaitConversation(known, 2, wait.timeoutMs, scenarioId); + } catch (error) { + note(`BR-10: the scenario coordinator's known delegate never answered — ${messageOf(error)}`); + } + if (lineBy(landed, firstDelegate).length !== 1) { + note(`BR-10: ${firstDelegate} is the scenario coordinator's delegate and did not answer`); } - // BR-17 — a post claiming an author the roster does not hold. - const before = (await scenario.readMailbox()).transcript.length; - const refused = await scenario.post(fixture.post, fixture.nonMemberAuthor); - if (refused.error === undefined || !refused.error.includes("author-not-a-member")) { - note(`BR-17: a non-member author was not refused by name (${refused.error ?? "accepted"})`); + // BR-17 — a post claiming an author. The door takes the operator's + // words and nothing else, so the claim is refused and nothing is written. + const before = (await known.readConversation()).lines.length; + const refused = await known.post(fixture.post, fixture.claimedAuthor); + if (refused.error === undefined || !refused.error.includes("author")) { + note(`BR-17: a post claiming an author was not refused naming the claim (${refused.error ?? "accepted"})`); } - const afterRefusal = (await scenario.readMailbox()).transcript.length; + const afterRefusal = (await known.readConversation()).lines.length; if (afterRefusal !== before) note(`BR-17: the refused post still wrote a line`); - // Its opposite state: a DECLARED member's claim is accepted. Without - // this, a mailbox that refuses every author passes the half above. - const accepted = await scenario.post(fixture.post, firstMember); + // Its opposite state: the same post, claiming nothing, is accepted. + // Without this, a door that refuses everything passes the half above. + const accepted = await known.post(fixture.post); if (accepted.error !== undefined) { - note(`BR-17: a declared member's own author claim was refused — ${accepted.error}`); + note(`BR-17: the same post claiming no author was refused — ${accepted.error}`); } + const afterAccepted = (await awaitConversation(known, before + 3).catch(async () => (await known.readConversation()).lines)).length; + if (afterAccepted <= before) note(`BR-17: the accepted post wrote no line`); } finally { - await scenario.dispose(); + await known.dispose(); } evidence.push( - `a mailbox naming a member that is no hired seat still opens, keeps it on the declared roster ` + - `and skips its delivery; a post claiming a non-member author is refused author-not-a-member ` + - `and writes nothing, while a declared member's claim is accepted` + `a coordinator naming a delegate no file declares is refused at load, naming both (${JSON.stringify(refusal.slice(0, 160))}); ` + + `the same coordinator without it opens and its known delegate answers; a post claiming an author is ` + + `refused naming the claim and writes nothing, while the same post claiming nothing is accepted` ); } } @@ -1017,7 +955,7 @@ await runGoal(async () => { { for (const [label, tree] of Object.entries(fixture.refusalTrees)) { // The bad tree. Nothing on the main run can produce this refusal — the - // ten-file tree is valid — so each case needs its own tree. + // tree is valid — so each case needs its own tree. { let hired: Lab | undefined; let refusal = ""; @@ -1080,9 +1018,9 @@ await runGoal(async () => { // ---- (i) BR-19 — a withheld answer fails loudly, naming the count -------- // - // The rule says delivery is best-effort, so a short transcript must never + // The rule says delivery is best-effort, so a short conversation must never // read as a pass. Nothing else on this run withholds an answer, so without - // this leg an implementation that treats a partial transcript as success + // this leg an implementation that treats a partial conversation as success // satisfies every other mutation here. { const held = await open({ dbFile: join(dir, "br19.db") }); @@ -1093,12 +1031,12 @@ await runGoal(async () => { // had landed in a second. let raised: unknown; try { - await awaitTranscript(held, 99, wait.shortTimeoutMs); + await awaitConversation(held, 99, wait.shortTimeoutMs); } catch (error) { raised = error; } - if (!(raised instanceof ShortTranscriptError)) { - note(`BR-19: a transcript short of its target did not fail loudly (${String(raised)})`); + if (!(raised instanceof ShortConversationError)) { + note(`BR-19: a conversation short of its target did not fail loudly (${String(raised)})`); } else { if (!/saw \d+ line\(s\), wanted 99/.test(raised.message)) { note(`BR-19: the failure does not name the observed line count: ${raised.message}`); @@ -1108,22 +1046,17 @@ await runGoal(async () => { // Its opposite state, on the same helper: a target the run does reach // returns rather than raising. Without this the check passes for a helper // that always throws. - const reached = await awaitTranscript(held, 1, wait.timeoutMs).catch( + const reached = await awaitConversation(held, 1, wait.timeoutMs).catch( (error) => error as Error ); if (reached instanceof Error) { note(`BR-19: the same helper failed on a target the run had reached — ${reached.message}`); } - // The OTHER bounded helper, both halves. - // - // `settledLength` used to return the last count it saw when its bound ran - // out, which reports a run whose deliveries are still in flight as a - // settled transcript — the caller could not tell a quiet mailbox from a - // slow one. No scenario control reaches that path (a withheld answer - // still settles, just later), so it is exercised here directly: a bound - // too short to observe any quiet period must raise, and an ordinary one - // must return. + // The OTHER bounded helper, both halves. No scenario control reaches its + // timeout path (a withheld answer still settles, just later), so it is + // exercised here directly: a bound too short to observe any quiet period + // must raise, and an ordinary one must return. let unsettled: unknown; try { await settledLength(held, 400, 1); @@ -1138,13 +1071,13 @@ await runGoal(async () => { } const quiet = await settledLength(held, 50, wait.timeoutMs).catch((error) => error as Error); if (quiet instanceof Error) { - note(`BR-19: settledLength failed on a transcript that had gone quiet — ${quiet.message}`); + note(`BR-19: settledLength failed on a conversation that had gone quiet — ${quiet.message}`); } } finally { await held.dispose(); } evidence.push( - "the shared bounded wait fails loudly on a short transcript, naming how many lines it saw, and " + + "the shared bounded wait fails loudly on a short conversation, naming how many lines it saw, and " + "returns on a target the run reaches" ); } diff --git a/goals/pentest-lab/a-seat-answers-from-its-own-document/fixtures/input.json b/goals/pentest-lab/a-seat-answers-from-its-own-document/fixtures/input.json index 8d0e8b2923..123e5ea0f5 100644 --- a/goals/pentest-lab/a-seat-answers-from-its-own-document/fixtures/input.json +++ b/goals/pentest-lab/a-seat-answers-from-its-own-document/fixtures/input.json @@ -1,6 +1,6 @@ { "post": "Sweep the target list and report what answered.", - "members": ["pentest.recon", "pentest.triage"], + "delegates": ["pentest.recon", "pentest.triage"], "toolSeat": "pentest.recon", "toollessSeat": "pentest.triage", "silentSeat": "audit.scribe", diff --git a/goals/pentest-lab/a-seat-answers-from-its-own-document/goal.md b/goals/pentest-lab/a-seat-answers-from-its-own-document/goal.md index a7e9dbc3c9..222cf5a538 100644 --- a/goals/pentest-lab/a-seat-answers-from-its-own-document/goal.md +++ b/goals/pentest-lab/a-seat-answers-from-its-own-document/goal.md @@ -1,12 +1,12 @@ # pentest-lab › it answers from its own document, with a real model -**Issue:** FIX-1355 (the pentest lab — the four file conventions, proved together; epic FIX-1351) +**Issue:** FIX-1355 (the pentest lab — the four file conventions, proved together; epic FIX-1351); converted by FIX-1792 P1 -**Outcome:** The same ten Markdown files, the same hire and the same mailbox as the gate beside this one — with a **generator** in the one answering slot instead of a handler. One operator post wakes both declared members; each answers in its own words, and each names the identifier that only its own document carries and the token that only its own `WORKER.md` carries, and neither its sibling's. The seat whose `tools:` names the lab's block calls it and reports a value that exists nowhere except inside that call. Its teammate, which names no tool, reports none of it and makes no call. +**Outcome:** The same ten Markdown files, the same hire and the same coordinator as the gate beside this one — with a **generator** in the one answering slot instead of a handler. One operator post to the coordinator reaches both declared delegates; each answers in its own words, and each names the identifier that only its own document carries and the token that only its own `WORKER.md` carries, and neither its sibling's. The seat whose `tools:` names the lab's block calls it and reports a value that exists nowhere except inside that call. Its teammate, which names no tool, reports none of it and makes no call. This is the **honesty check**. The gate proves the plumbing carried every value; it cannot prove the values were any use, because a handler printing its own config proves plumbing rather than usefulness (epic ER-19). Required at completion, not optional ([D2](../../../spec/FIX-1355/DECISIONS.md)) — a red BR-20 is this issue unfinished, and only an unavailable inference credential is recorded **blocked**. -**Input:** `fixtures/input.json` — the two members, which seat holds the tool, the document each reads and the held-out token each of those files carries. The tree is `../lab/workforce/`, shared with the gate. Held-out in the same strict sense: each token lives in exactly one convention file and nowhere in the lab's code or the post, which the gate's leg 0 scans the whole tree to prove. +**Input:** `fixtures/input.json` — the two delegates, which seat holds the tool, the document each reads and the held-out token each of those files carries. The tree is `../lab/workforce/`, shared with the gate. Held-out in the same strict sense: each token lives in exactly one convention file and nowhere in the lab's code or the post, which the gate's leg 0 scans the whole tree to prove. **The one deliberate inversion, and the reason this goal exists.** Every other marker must come from a file. The `scope-check` tool's **clearance value is minted in code at construction** and appears in no file, no frontmatter and no prompt — because this leg's claim is the opposite one. A marker a *file* could have supplied would prove nothing was called. Its value is different every run, so a stale transcript cannot pass a fresh check. @@ -16,14 +16,14 @@ This is the **honesty check**. The gate proves the plumbing carried every value; - **BR-4 / BR-20** — each answer names the identifier only its own document carries, and **not** its sibling's document token. The absent half is what makes "only" mean anything. - **BR-12** — the seat-exclusive half only: a worker-level skill token must not appear in the sibling's answer. `report-format` and `port-scan` are **not** excluded — both pentest seats are one team and legitimately read them, and excluding them would fail a correct run. - **BR-21** — the tool seat's line carries the minted clearance value; the tool-less seat's carries none of it; and the recorded calls are attributed to the tool seat **alone**. That a seat's config *lists* the tool is never what is graded. -- **BR-13 / BR-14** — the silent seat is never dispatched to; the transcript is graded on presence and attribution, never index. +- **BR-13 / BR-14** — the silent seat is never handed the post (the coordinator's routing record); the conversation is graded on presence and attribution, never index. **Anti-game:** a hollow pass would assert that two answers arrived and each mentioned its own token — true of a seat that carries both its own and its sibling's, which is exactly the failure a handler cannot produce and a generator can. So the check: - MUST assert the **absent** half of every rule whose text says a value must be absent, and MUST restrict that exclusion to values the seat is genuinely not entitled to. Two questions, always both: *is the absent half asserted?* and *could this exclusion catch a value the seat is legitimately entitled to?* - MUST grade BR-21 on the **minted value and the recorded call**, never on the `tools:` setting. A setting is not a call. -- MUST assert the transcript reached three lines and the fan-out addressed both members **before** any "must not carry" clause, since all of those are vacuously true of a transcript nothing ever wrote to. -- MUST NOT retry the mechanism, and the structure has to make that true rather than the prose asserting it. **One harness, N rounds inside it:** the tree is read, the roster hired and the mailbox opened once; a round repeats only the post, the fan-out and the model's answer, and the tool's clearance value is minted once for all of them. Grading is split to match — **mechanism clauses must hold in every round** (a wiring failure fails them all identically and no later round can hide it), while only the **answer clauses** need hold in one. A fresh harness per attempt re-initialises everything and mints a new clearance each time, so a first-round mechanism failure can be papered over by a later one. +- MUST assert the conversation reached three lines and the coordinator delivered to both delegates **before** any "must not carry" clause, since all of those are vacuously true of a conversation nothing ever wrote to. +- MUST NOT retry the mechanism, and the structure has to make that true rather than the prose asserting it. **One harness, N rounds inside it:** the tree is read, the roster hired and the conversation opened once; a round repeats only the post, the delivery and the model's answer, and the tool's clearance value is minted once for all of them. Grading is split to match — **mechanism clauses must hold in every round** (a wiring failure fails them all identically and no later round can hide it), while only the **answer clauses** need hold in one. A fresh harness per attempt re-initialises everything and mints a new clearance each time, so a first-round mechanism failure can be papered over by a later one. **Model:** real — `openai/gpt-5.4-mini`, gateway-qualified by the harness's own resolver (`vercel/openai/gpt-5.4-mini`). Override with `GOAL_MODEL`. **One round by default** (two model calls); `GOAL_ATTEMPTS=3` buys flake tolerance for the model's phrasing, and all three rounds run inside the same lab. The harness runs every round it is asked for, because it reports observations and never grades — it cannot know whether a round satisfied the runner, so it cannot stop early. @@ -44,3 +44,5 @@ This is the **honesty check**. The gate proves the plumbing carried every value; | 2026-10-01 | `f5f4d9342` (FIX-1695 / FIX-1443) | vercel/openai/gpt-5.4-mini | **PASS** | Red on `main` at `a25134ebd` at the mailbox read (the org binding, see the gate's log). With the lab's org named at `resolvePrincipal`: one round, both declared members woke, each answer named its own instructions token and only its own document's identifier, and `pentest.recon` called the tool once and carried its minted clearance. | | 2026-10-01 | `f5f4d9342` | vercel/openai/gpt-5.4-mini | FAIL (expected) | `tool-dropped` and `handler-in-the-slot` each red on *the tool was never called* and the missing clearance. `documents-swapped` red on both halves (each seat misses its own identifier; `recon` carries `triage`'s `ROE-TOKEN-8F31A`) on its second run; its first run's fan-out never settled with 0 lines, the latency case the FIX-1443 spec names, and was re-run once. | | 2026-10-01 | `10acd61ea` | vercel/openai/gpt-5.4-mini | **PASS** | Re-run after the host's review fixes. The first run's fan-out never settled with 0 lines (the latency case); the re-run passed. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | vercel/openai/gpt-5.4-mini | **PASS** | **Converted (FIX-1792 P1)**: the post goes to the coordinator `pentest.findings`, and each answer is read from the operator's conversation under its delegate's name; who was handed the post is the coordinator's routing record. One round: both delegates reached, each answer named its own instructions token and only its own document's identifier, `pentest.recon` called `scope-check` once and carried `CLEARANCE-07D1B385`, `pentest.triage` none of it and no call. | +| 2026-10-09 | same | vercel/openai/gpt-5.4-mini; n/a for `handler-in-the-slot` | FAIL (expected) | All three controls. `tool-dropped` and `handler-in-the-slot` each red on *the tool was never called* and the missing clearance. `documents-swapped` red on the presence half: neither seat names the identifier its own document should carry. | diff --git a/goals/pentest-lab/a-seat-answers-from-its-own-document/harness.mts b/goals/pentest-lab/a-seat-answers-from-its-own-document/harness.mts index b32368e906..24431dd8af 100644 --- a/goals/pentest-lab/a-seat-answers-from-its-own-document/harness.mts +++ b/goals/pentest-lab/a-seat-answers-from-its-own-document/harness.mts @@ -101,11 +101,12 @@ const lab = await openLab({ }); /** - * One round: post, and wait for the three lines that post produces. + * One round: post to the coordinator, and wait for the three lines that post + * produces in the operator's conversation. * * **Rounds share one lab, and that is the whole point.** The tree is read, the - * roster hired and the mailbox opened exactly once, above; a round repeats only - * the post, the fan-out and the model's answer. So a wiring failure fails every + * roster hired and the conversation opened exactly once, above; a round + * repeats only the post, the delivery and the model's answer. So a wiring failure fails every * round identically and cannot be hidden by a later one, while a model that * phrased an answer badly can be given another go — which is the difference * between retrying the model and retrying the mechanism. @@ -119,20 +120,21 @@ async function round(): Promise<{ addressed: string[]; settled: boolean; }> { - const lineBase = (await lab.readMailbox()).transcript.length; + const before = await lab.readConversation(); + const lineBase = before.lines.length; + const routeBase = before.routes.length; const callBase = scopeCheck.calls.length; - const addressedBase = lab.notifyLog.addressed.length; await lab.post(POST); // The bounded wait, with named bounds — the gate's rule, applied here too. A - // short transcript is reported as what it is and never as a pass. - let transcript: any[] = []; + // short conversation is reported as what it is and never as a pass. + let read = before; let settled = false; const deadline = Date.now() + TIMEOUT_MS; while (Date.now() < deadline) { - transcript = (await lab.readMailbox()).transcript; - if (transcript.length >= lineBase + 3 && (await lab.fanOutSettled())) { + read = await lab.readConversation(); + if (read.lines.length >= lineBase + 3 && (await lab.settled())) { settled = true; break; } @@ -140,16 +142,19 @@ async function round(): Promise<{ } return { - // This round's own lines: the tail the round appended. Sliced by index on an - // append-only array, which says nothing about the ORDER the two seats - // landed in — only which round they belong to. - lines: transcript.slice(lineBase).map((line: any) => ({ - author: line.author ?? null, - authorVerified: line.authorVerified, + // This round's own lines: the tail the round appended. Sliced by index on + // the conversation's lines, which says nothing about the ORDER the two + // seats landed in — only which round they belong to. + lines: read.lines.slice(lineBase).map((line: any) => ({ + author: line.author, + role: line.role, body: line.body, })), toolCalls: scopeCheck.calls.slice(callBase), - addressed: lab.notifyLog.addressed.slice(addressedBase), + // Who the coordinator delivered this round's post to, off its routing record. + addressed: read.routes + .slice(routeBase) + .flatMap((record: any) => record.delegates.filter((d: any) => d.outcome === "delivered").map((d: any) => d.worker)), settled, }; } @@ -159,13 +164,13 @@ for (let attempt = 0; attempt < ATTEMPTS; attempt += 1) { rounds.push(await round()); } +const routes = (await lab.readConversation()).routes; const observations = { model: MODEL, control: CONTROL === "" ? null : CONTROL, tree: LAB_TREE, clearance: scopeCheck.clearance, - skipped: lab.notifyLog.skipped, - refusals: lab.notifyLog.refusals, + skipped: routes.flatMap((record: any) => record.delegates.filter((d: any) => d.outcome === "skipped").map((d: any) => d.worker)), rounds, }; diff --git a/goals/pentest-lab/a-seat-answers-from-its-own-document/run.mts b/goals/pentest-lab/a-seat-answers-from-its-own-document/run.mts index 335d985ece..334fed661c 100644 --- a/goals/pentest-lab/a-seat-answers-from-its-own-document/run.mts +++ b/goals/pentest-lab/a-seat-answers-from-its-own-document/run.mts @@ -28,6 +28,10 @@ * `@ai-sdk/gateway` resolves from an app's `node_modules` and not from `goals/`. * It reports raw observations; every assertion below is this file's. * + * The post goes to the tree's coordinator, `pentest.findings`, which routes + * to everyone: both declared delegates take it, and each answer lands in the + * operator's conversation under the delegate's name. + * * Required at completion, not optional (D2): a red BR-20 is this issue * unfinished, and only an unavailable inference credential is recorded blocked. * @@ -82,7 +86,7 @@ interface SeatFixture { interface Fixture { post: string; - members: string[]; + delegates: string[]; toolSeat: string; toollessSeat: string; silentSeat: string; @@ -93,10 +97,12 @@ interface Fixture { timeoutMs: number; } -/** What one round of post → fan-out → answer produced, inside the shared lab. */ +/** What one round of post → delivery → answer produced, inside the shared lab. */ interface Round { - lines: Array<{ author: string | null; authorVerified: boolean; body: string }>; + /** The round's lines in the operator's conversation: its post, and each answer under its delegate's name. */ + lines: Array<{ author: string | null; role: string; body: string }>; toolCalls: Array<{ by: string; host: string }>; + /** The delegates the coordinator delivered this round's post to, off its routing record. */ addressed: string[]; settled: boolean; } @@ -105,8 +111,8 @@ interface Observation { model: string; control: string | null; clearance: string; + /** Every delegate a routing record skipped, with why. */ skipped: string[]; - refusals: string[]; rounds: Round[]; } @@ -138,8 +144,8 @@ const lineOf = (round: Round, author: string): string | undefined => * The clauses that are about the WIRING, not about what the model said. * * These must hold in **every** round. They are the half a retry must never be - * able to paper over: the tree was read, the roster hired, the mailbox opened, - * the fan-out dispatched, and the tool was reached. A round that fails one of + * able to paper over: the tree was read, the roster hired, the conversation + * opened, the coordinator delivered, and the tool was reached. A round that fails one of * these failed for a reason another round cannot fix. */ function gradeMechanism(round: Round, index: number, seen: Observation): string[] { @@ -147,23 +153,23 @@ function gradeMechanism(round: Round, index: number, seen: Observation): string[ const at = `round ${index + 1}`; if (!round.settled) { - failures.push(`${at}: the fan-out never settled, so this round proves nothing`); + failures.push(`${at}: the deliveries never settled, so this round proves nothing`); } // Positive first, and not as decoration: every "must not carry" clause below // is trivially true of a round that produced nothing at all. if (round.lines.length !== 3) { failures.push( - `${at}: produced ${round.lines.length} line(s), wanted 3 — the operator's and one per member` + `${at}: produced ${round.lines.length} line(s), wanted 3 — the operator's and one per delegate` ); } - if ([...round.addressed].sort().join(",") !== [...fixture.members].sort().join(",")) { - failures.push(`${at}: the fan-out addressed ${JSON.stringify(round.addressed)}, wanted both members`); + if ([...round.addressed].sort().join(",") !== [...fixture.delegates].sort().join(",")) { + failures.push(`${at}: the coordinator delivered to ${JSON.stringify(round.addressed)}, wanted both delegates`); } - for (const member of fixture.members) { - if (lineOf(round, member) === undefined) failures.push(`${at}: ${member} put no line in this round`); + for (const delegate of fixture.delegates) { + if (lineOf(round, delegate) === undefined) failures.push(`${at}: ${delegate} put no line in this round`); } if (round.addressed.includes(fixture.silentSeat)) { - failures.push(`${at}: the fan-out dispatched to ${fixture.silentSeat}`); + failures.push(`${at}: the coordinator delivered to ${fixture.silentSeat}`); } // The tool was reached, and by the right seat. Whether its VALUE made it into @@ -189,12 +195,12 @@ function gradeMechanism(round: Round, index: number, seen: Observation): string[ */ function gradeAnswers(round: Round, seen: Observation): string[] { const failures: string[] = []; - const [first, second] = fixture.members as [string, string]; + const [first, second] = fixture.delegates as [string, string]; - for (const member of fixture.members) { + for (const member of fixture.delegates) { const body = lineOf(round, member); if (body === undefined) { - failures.push(`${member} put no line in the transcript`); + failures.push(`${member} put no line in the conversation`); continue; } @@ -285,12 +291,14 @@ await runGoal(async () => { try { await lab.post(fixture.post); const deadline = Date.now() + 30_000; - let transcript: Array<{ author?: string; authorVerified: boolean; body: string }> = []; + let read = await lab.readConversation(); while (Date.now() < deadline) { - transcript = (await lab.readMailbox()).transcript; - if (transcript.length >= 3 && (await lab.fanOutSettled())) break; + read = await lab.readConversation(); + if (read.lines.length >= 3 && (await lab.settled())) break; await new Promise((resolve) => setTimeout(resolve, 50)); } + const outcomes = (outcome: string) => + read.routes.flatMap((record) => record.delegates.filter((d) => d.outcome === outcome).map((d) => d.worker)); // Graded through the SAME two graders the model path uses, on a single // round — otherwise this control would be exercising a different check // from the one it exists to red-state. @@ -298,18 +306,13 @@ await runGoal(async () => { model: "n/a", control: CONTROL, clearance: scopeCheck.clearance, - skipped: lab.notifyLog.skipped, - refusals: lab.notifyLog.refusals, + skipped: outcomes("skipped"), rounds: [ { - lines: transcript.map((line) => ({ - author: line.author ?? null, - authorVerified: line.authorVerified, - body: line.body, - })), + lines: read.lines.map((line) => ({ author: line.author, role: line.role, body: line.body })), toolCalls: scopeCheck.calls, - addressed: lab.notifyLog.addressed, - settled: transcript.length >= 3, + addressed: outcomes("delivered"), + settled: read.lines.length >= 3, }, ], }; @@ -331,8 +334,8 @@ await runGoal(async () => { // ---- the model runs ---------------------------------------------------- // // **One harness, N rounds inside it.** The tree is read, the roster hired and - // the mailbox opened once; a round repeats only the post, the fan-out and the - // model's answer. That split is what lets the two graders mean different + // the conversation opened once; a round repeats only the post, the delivery + // and the model's answer. That split is what lets the two graders mean different // things: // // - mechanism clauses must hold in EVERY round. A wiring failure fails them @@ -341,7 +344,7 @@ await runGoal(async () => { // it is the only part worth another go. // // A fresh harness per attempt — which is what this did before — re-read the - // tree, re-hired, re-opened the mailbox and minted a new clearance value each + // tree, re-hired, re-opened the conversation and minted a new clearance value each // time, so a first-round MECHANISM failure could be papered over by a later // round. This goal's own anti-game forbids exactly that. const seen = drive(); @@ -368,8 +371,8 @@ await runGoal(async () => { return { failures, evidence: - `model ${seen.model}, ${seen.rounds.length} round(s) in one lab: every round woke exactly the ` + - `two declared members and reached the tool, and round ${passed + 1}'s answers each name their ` + + `model ${seen.model}, ${seen.rounds.length} round(s) in one lab: every round reached exactly the ` + + `two declared delegates and the tool, and round ${passed + 1}'s answers each name their ` + `own instructions token and the identifier only their own document carries, and neither ` + `sibling's; ${fixture.toolSeat} called ${calls} time(s) and its line carries ${seen.clearance}, ` + `the value that block minted at call time, while ${fixture.toollessSeat} — which declares no ` + diff --git a/goals/pentest-lab/lab/README.md b/goals/pentest-lab/lab/README.md index 4c2c8ad115..89a00aab2f 100644 --- a/goals/pentest-lab/lab/README.md +++ b/goals/pentest-lab/lab/README.md @@ -1,9 +1,9 @@ # The pentest lab -Four file conventions — workers, skills, resources, mailboxes — each shipped with -a green suite of its own. Nothing ran them together. This is the thing that -does: one tree of Markdown, pointed at once, ending as a transcript two seats -wrote. +Three file conventions — workers, skills, resources — each shipped with a green +suite of its own, and a coordinator is a worker file too. Nothing ran them +together. This is the thing that does: one tree of Markdown, pointed at once, +ending as a conversation with a coordinator that two seats answered in. Two checks drive it, and they differ by **one block**: @@ -29,19 +29,21 @@ workforce/ pentest/ resources/scope.md this team's target list skills/port-scan/SKILL.md - mailboxes/findings/MAILBOX.md members: recon, triage + workers/findings/WORKER.md flow: coordinator, delegates: recon, triage workers/recon/WORKER.md tools: [scope-check] workers/recon/skills/sweep/SKILL.md recon's alone workers/triage/WORKER.md audit/ - workers/scribe/WORKER.md not a member — the silent probe + workers/scribe/WORKER.md not a delegate — the silent probe skills/port-scan/SKILL.md the same NAME, another team ``` -The kind is one file under `workforce/flows/workers/`; a mailbox kind would -sit beside it under `workforce/flows/mailboxes/`. Ten Markdown files after -that, and **none of those is named anywhere in this directory's code** except -the root of the tree. Everything else is walked. +The kind is one file under `workforce/flows/workers/`. The coordinator, +`pentest.findings`, is a worker file on Workforce's own `coordinator` flow: +`delegates:` names who it hands a post to, and `routing: everyone` hands each +post to all of them. Ten Markdown files after that, and **none of those is +named anywhere in this directory's code** except the root of the tree. +Everything else is walked. Each file carries one held-out token that appears in **exactly one** convention file and nowhere in this code or in the operator's post. The gate scans the @@ -58,33 +60,32 @@ on bodies rather than names. ## What this directory adds, and why each piece is here -Four things, and each exists because the framework has no opinion at that spot — +Three things, and each exists because the framework has no opinion at that spot — not because a convention is missing. | File | What it is | Why it is the lab's | |---|---|---| -| [`workforce/flows/workers/probe.mts`](workforce/flows/workers/probe.mts) | The `probe` worker kind every `WORKER.md` names | A mailbox delivery is an `internal` dispatch, and the built-in `agent` kind declares no internal map at all, so nothing can wake it. What this proves is the *convention* surface on a kind the lab wrote. Custom kinds live file-per-kind here; the basename is the kind id. | -| [`notify.mts`](notify.mts) | The block passed as `defineMailboxFlow({ notify })` | Not a second fan-out: `mailbox-flow.ts` keeps the member walk and runs this once per member. Only the **policy** is here — which posts route, and what address each member resolves to — because the dispatch seam refuses a target taken out of stored data. | -| [`host.mts`](host.mts) | Read the tree, build the kinds, hire, open the mailboxes | The assembly an app writes anyway. | +| [`workforce/flows/workers/probe.mts`](workforce/flows/workers/probe.mts) | The `probe` worker kind every seat's `WORKER.md` names | What this proves is the *convention* surface on a kind the lab wrote, not on the built-in `agent`. It takes a coordinator's post on the internal entry Workforce's `delegatedPostEntry` declares, the way the built-in kind does. Custom kinds live file-per-kind here; the basename is the kind id. | +| [`host.mts`](host.mts) | Read the tree, build the flows, hire, open the operator's conversation with each coordinator | The assembly an app writes anyway: the lab's kind and Workforce's `coordinator` flow, registered on one worker installation. | | [`scope-check.mts`](scope-check.mts) | One tool block, key `scope-check` | So that a seat's `tools:` fence can be graded on a **call** rather than on a setting. | ## Two things worth knowing before you change anything -**The cycle break is the app's to write.** A seat's answer is a post like any -other, and `mailbox-flow.ts` taps the hand-off for *every* post. Without the -author gate in `notify.mts`, two seats answer each other without end — -1 → 2 → 4 → 8 detached dispatches with nothing to stop them. Nothing in the -framework prevents this today, which is why the rule is written here in the open -rather than assumed. +**An answer goes no further.** A seat's answer lands in the operator's +conversation under the seat's name, and the coordinator hands it to nobody: +`rounds:` is 0 unless the file says otherwise. Raise it and each answer goes +back out to the other delegates, once per round. The gate's `answers-go-on` +control is exactly that, and it fails. **The org is named once, where the host resolves who is calling.** `openLab` gives `createFlowState` a `resolvePrincipal` that answers "the lab's user, in the -lab's org". A mailbox session is bound to the org of the caller that created it, -never to anything in the request body, so this is what puts the mailbox and the -seats in the same org. Every file-declared document is org-scoped, so a lab -without it opens a mailbox under the development org that its own seats are then -refused delivery into, by name. Leaving the resolver out is one of the gate's -controls (`GOAL_CONTROL=no-principal-org`), and it fails at the first read. +lab's org". A session is bound to the org of the caller that created it, never +to anything in the request body, so this is what puts the operator's +conversation and the seats in the same org. Every file-declared document is +org-scoped, so a lab without it opens the conversation under the development org +and its own reads are then refused, by name. Leaving the resolver out is one of +the gate's controls (`GOAL_CONTROL=no-principal-org`), and it fails at the first +read. ## What it works around, rather than patches diff --git a/goals/pentest-lab/lab/fsdev.config.mts b/goals/pentest-lab/lab/fsdev.config.mts index 2ab0d90ca6..9be6681a11 100644 --- a/goals/pentest-lab/lab/fsdev.config.mts +++ b/goals/pentest-lab/lab/fsdev.config.mts @@ -5,9 +5,9 @@ * * One `FlowState`, default-exported, opened through the same `openLab` the * lab's checks use, so a server and a check boot the same tree the same way. - * What a server asks of it that a check doesn't: the organization's seat and - * mailbox inventory, opened after the mailboxes (Shift Manager's TEAMS and PROJECTS - * read it), and the lab's user handed to the page through the `devtool` block. + * What a server asks of it that a check doesn't: the organization's seat + * inventory (Shift Manager's TEAMS reads it), written from a flow of the lab's + * own, and the lab's user handed to the page through the `devtool` block. * With no `answer`, a seat answers with the host's model-free line. The stores * are in memory, so every start is fresh. */ diff --git a/goals/pentest-lab/lab/host.mts b/goals/pentest-lab/lab/host.mts index f0c1f4085e..8d042cf8b1 100644 --- a/goals/pentest-lab/lab/host.mts +++ b/goals/pentest-lab/lab/host.mts @@ -4,38 +4,43 @@ * differ by one block. * * What `openLab` does, in order, and nothing else: read the tree, build the - * kinds, hire, register, open the mailboxes (and, for a server, the - * organization's inventory), hand back the handles. Every file it reads is - * found by walking from one root; no file is named in this code. + * flows, hire, register, open the operator's conversation with each + * coordinator (and, for a server, the organization's inventory), hand back the + * handles. Every file it reads is found by walking from one root; no file is + * named in this code. * - * Four pieces here are the lab's rather than the framework's, each because the - * framework has no opinion at that spot — not because a convention is missing: - * the worker kind, the notify router's policy, this host, and the one tool - * block a seat names in its `tools:` fence. + * The coordinator is a worker file (`flow: coordinator`), and the lab + * registers Workforce's `coordinator` flow for it, with the lab's `probe` + * kind as the flow its delegates take posts on. Three pieces here are the + * lab's rather than the framework's, each because the framework has no + * opinion at that spot: the worker kind, this host, and the one tool block a + * seat names in its `tools:` fence. * - * The fifth thing: **the lab names its org at the host's `resolvePrincipal`**, + * The fourth thing: **the lab names its org at the host's `resolvePrincipal`**, * the seam every app uses to say who a request is. A session's org comes from - * the caller that created it, so this is what binds the mailbox to the same org - * the seats act in. Leaving it out is the gate's `no-principal-org` control, and - * it fails. + * the caller that created it, so this is what binds the operator's + * conversation to the same org the seats act in. Leaving it out is the gate's + * `no-principal-org` control, and it fails. */ -import { createFlowState, ensureSessionRecord, runAction, type FlowState, type StoreRegistry } from "@flow-state-dev/engine"; -import { handler } from "@flow-state-dev/core"; +import { createFlowState, ensureSessionRecord, runAction, type FlowState } from "@flow-state-dev/engine"; +import { defineFlow, handler } from "@flow-state-dev/core"; import type { BlockDefinition, FlowInstance } from "@flow-state-dev/core/types"; import { z } from "zod"; import { - MAILBOX_KIND, - mailboxInstances, - defineMailboxFlow, + COORDINATOR_KIND, + COORDINATOR_ROUTE, + LIST_DELEGATES, createWorkerInstallation, + defineCoordinatorFlow, hireWorkforce, inventorySeats, - openMailboxes, + inventoryWriterActions, openInventory, resourcesFromDocs, - type MailboxManifest, + type CoordinatorRouteRecord, type InventoryActionRequest, + type WorkerManifest, } from "@flow-state-dev/workforce"; import { readDeclaredRoster, @@ -48,16 +53,32 @@ import { seatFactsSchema, type SeatFacts, } from "./workforce/flows/workers/probe.mts"; -import { createNotifyLog, labNotify, type NotifyLog } from "./notify.mts"; import { createScopeCheck, type ScopeCheck } from "./scope-check.mts"; /** The authored tree — the one path this code names. Everything else is walked. */ export const LAB_TREE = fileURLToPath(new URL("./workforce", import.meta.url)); -/** Who the lab runs as, and the org every document and mailbox session is bound to. */ +/** Who the lab runs as, and the org every document and session is bound to. */ export const LAB_USER_ID = "u_pentest_lab"; export const LAB_ORG_ID = "org_pentest_lab"; +/** + * The kind a seat moves to under the gate's `noPostFor` option: the probe + * kind again, under another name, and not one the coordinator delivers posts + * on. The tree names no such kind. + */ +export const DESK_KIND = "probe-desk"; + +/** + * The model best fit's one evaluator call would run on. The lab's coordinators + * route to everyone and never make that call, but the flow names no default, + * so the lab names one. + */ +const ROUTE_MODEL = "vercel/openai/gpt-5.4-mini"; + +/** The flow the inventory's seat rows are written from: the lab's own, carrying the writer. */ +const INVENTORY_WRITER_KIND = "inventory-writer"; + /** * Read a workforce tree into records, refusing a tree that did not load * cleanly. @@ -68,7 +89,7 @@ export const LAB_ORG_ID = "org_pentest_lab"; * refusal is this lab's own policy, which is why it lives here and not behind * the export. */ -async function loadTree(root: string): Promise { +export async function loadTree(root: string): Promise { const roster = await readDeclaredRoster(root); if (roster.problems.length > 0) { const lines = roster.problems.map((p) => @@ -81,6 +102,10 @@ async function loadTree(root: string): Promise { return roster; } +/** Whether a worker file names the coordinator flow. */ +export const isCoordinator = (worker: { declared: Record }): boolean => + worker.declared.flow === COORDINATOR_KIND; + /** * The answering block a server runs when no check supplies one: no model, one * line naming the seat and the document it read. @@ -100,7 +125,7 @@ export interface OpenLabOptions { * The adapter rather than a resolved registry: `createFlowState` fills its * capability slots from adapters, and it is what `dispose()` then closes. That * is BR-15's whole mechanism — dispose this lab, build another over the same - * filename, and read the transcript back out of the file. + * filename, and read the conversation back out of the file. */ stores: unknown; /** @@ -125,16 +150,16 @@ export interface OpenLabOptions { modelResolver?: unknown; /** The tree to read. Defaults to the lab's own. */ root?: string; - /** Extra mailbox records to open beside the tree's — the BR-10 scenario's. */ - extraMailboxes?: MailboxManifest[]; + /** Extra worker records hired beside the tree's — the BR-10 scenario's coordinator. */ + extraWorkers?: WorkerManifest[]; /** The model a seat that names none runs on. */ defaultModel?: string; /** Silence the engine's own logging. */ logger?: unknown; /** - * Open the organization's seat and mailbox inventory once the mailboxes are - * open, which is what Shift Manager's TEAMS and PROJECTS read. The mailbox kind is - * then built with the inventory's writer. The checks leave it off. + * Open the organization's seat inventory once the conversations are open, + * which is what Shift Manager's TEAMS reads. The lab then registers a flow of + * its own carrying the inventory's writer. The checks leave it off. */ inventory?: boolean; /** @@ -147,45 +172,46 @@ export interface OpenLabOptions { /** * Leave the host's `resolvePrincipal` out — the `no-principal-org` control. - * The mailbox session then binds to the development org while every action - * still runs as `org_pentest_lab`, and the first read is refused by name. + * The conversation session then binds to the development org while every + * action still runs as `org_pentest_lab`, and the first read is refused by + * name. */ omitPrincipal?: boolean; - /** Leave these declared members out of the router's address map — BR-9. */ - omitAddressFor?: string[]; /** - * Append these ids to the tree mailbox's declared `members` before it is - * bound — BR-13's control. - * - * It has to be membership rather than an address: `mailbox-flow.ts` walks the - * mailbox's own declared roster and never hands the notify block a delivery - * for anyone else, so giving a non-member an address changes nothing. Adding - * `audit.scribe` here is what makes "the fan-out never woke it" able to go - * red. + * Move these delegates onto {@link DESK_KIND}, a flow the coordinator does + * not deliver posts on — BR-9. The record still names them as delegates; a + * post can't reach them, and the routing record says so. */ - alsoMember?: string[]; + noPostFor?: string[]; /** - * Bind the tree's mailbox with its `MAILBOX.md` body dropped — the charter - * leg's control. + * Append these ids to the tree coordinator's declared `delegates` before the + * installation is built — BR-13's control. * - * The roster still binds and the mailbox still opens; only the charter is - * gone. That is precisely the failure the charter assertion exists to catch, - * and precisely the one a check reading only the id and the members cannot - * see. + * It has to be the record's delegates: the coordinator hands a post only to + * its own delegates, so anything short of naming `audit.scribe` there + * changes nothing. Adding it is what makes "the coordinator never delivered + * to it" able to go red. + */ + alsoDelegate?: string[]; + /** + * Hire the tree's coordinator with its `WORKER.md` body dropped — the + * charter clause's control. The installation still builds and the post + * still routes; only the charter is gone. */ dropCharter?: boolean; - /** Route posts that already carry an author — V2's control, and BR-8's red state. */ - routeAuthoredLines?: boolean; - /** Suppress the seats' post-backs — the other half of V2's control. */ - suppressPostBack?: boolean; + /** + * Hire the tree's coordinator with `rounds:` set — BR-8's control. An answer + * then goes back out to the other delegates instead of stopping. + */ + coordinatorRounds?: number; /** * Swap or redirect which document a seat reads, by seat id — VM's and VG's - * documents-swapped control. Applied to the record before the mint, so the + * documents-swapped control. Applied to the record before the hire, so the * seat genuinely runs on a different ref rather than being graded differently. */ documentOverrides?: Record; /** - * Rewrite one seat's `tools:` fence before the mint — the model-backed + * Rewrite one seat's `tools:` fence before the hire — the model-backed * check's first control. * * Dropping `scope-check` from the seat that names it means the clearance @@ -195,7 +221,7 @@ export interface OpenLabOptions { */ toolOverrides?: Record; /** - * Rewrite one seat's resolved skill union before the mint — BR-3's two + * Rewrite one seat's resolved skill union before the hire — BR-3's two * seat-side controls. * * Applied to the record, so the seat genuinely hires with the mutated set and @@ -214,79 +240,100 @@ export interface SeatSkill { files?: Array<{ path: string; content: string }>; } +/** + * One line of the operator's conversation with a coordinator: a stored + * `message` item, whole. + * + * `item` carries every field the store holds, not just the ones a reader looks + * at: BR-15 compares complete records across a restart, and a type that + * kept only the author and the body would quietly narrow that comparison to + * the fields it happened to name. + */ +export interface ConversationLine { + id: string; + /** `user` for the operator's post; `assistant` for an answer. */ + role: string; + /** The delegate an answer is drawn under, as the coordinator wrote it; `null` for the operator's post. */ + author: string | null; + body: string; + /** The stored item, every field. */ + item: Record; +} + +/** What one coordinator's operator conversation holds, read back through the real routes. */ +export interface ConversationRead { + /** The conversation's delegates, through the coordinator's own `listDelegates`. */ + delegates: string[]; + lines: ConversationLine[]; + /** The `coordinator-route` records: one per routing decision, oldest first. */ + routes: CoordinatorRouteRecord[]; +} + /** Everything a check needs to drive and observe one lab. */ export interface Lab { /** The lab as a server: what `fsdev.config.mts` default-exports. */ state: FlowState; roster: DeclaredRoster; - /** The hired seats, by id. */ + /** The hired seats, by id: every worker the tree puts on a lab kind, each to its flow's one copy. */ seats: Record; - /** The mailbox session ids the lab opened, in id order. */ - mailboxIds: string[]; - /** The tree's own mailbox — the one the two members are declared in. */ - mailboxId: string; - /** What the router decided, per delivery. */ - notifyLog: NotifyLog; - /** The tool, its minted clearance value, and the calls it actually received. */ - scopeCheck: ScopeCheck; + /** Every coordinator the lab hired, in id order. */ + coordinatorIds: string[]; + /** The tree's own coordinator — the one the two seats are declared delegates of. */ + coordinatorId: string; /** - * Post one line. With no `author` it is the operator's, which is what the - * fan-out routes; with one it is a seat's, which BR-8 says routes to nobody. - * - * Returns the refusal rather than throwing: BR-17's refused post is a - * per-request refusal the run is meant to survive and grade, not an error. + * A coordinator worker as the installation hands it to the `coordinator` + * flow: the record its turn is configured from. Its body is the charter. */ - post(body: string, author?: string, mailboxId?: string): Promise<{ error?: string }>; - /** Read a mailbox's members and transcript back through its own `read` action. */ - readMailbox(mailboxId?: string): Promise<{ members: string[]; transcript: TranscriptLine[] }>; + coordinatorWorker(id?: string): { delegates: string[]; instructions: string; description: string | null }; + /** The tool, its minted clearance value, and the calls it actually received. */ + scopeCheck: ScopeCheck; /** - * Read a mailbox session's stored state — members, **charter** and transcript. + * Post one line to a coordinator, as the operator, through its own door. * - * Separate from `readMailbox` because the mailbox's own `read` action projects - * `{ id, description, members, transcript }` and deliberately does **not** - * surface `instructions`. The charter is the `MAILBOX.md` body, and BR-5 - * claims it reaches the opened session, so it needs a route that can see it: - * the same session read `openMailboxes` answers a 409 with. + * `author` adds an author claim to the input, which the door has no field + * for: BR-17's refused post. Returns the refusal rather than throwing, since + * that refusal is a per-request one the run is meant to survive and grade. */ - mailboxState(mailboxId?: string): Promise<{ - members: string[]; - instructions: string; - transcript: TranscriptLine[]; - }>; + post(body: string, author?: string, coordinatorId?: string): Promise<{ error?: string }>; + /** Read the operator's conversation with a coordinator back. */ + readConversation(coordinatorId?: string): Promise; + /** The workers a coordinator's conversation opened a delegate session for, read off its child sessions. */ + delegateSessions(coordinatorId?: string): Promise; /** - * Read one seat's own view of itself, directly — not a wakening by the fan-out. + * Read one seat's own view of itself, directly — not a delivery from the + * coordinator. * * `omitOrg` sends the request with no org at all, which BR-16 says is * refused at `runAction`, the door every action passes through — * organization is unconditional, so there is no declaration to opt in or - * out of. The refusal is returned rather than - * thrown, so it can be graded on its wording. + * out of. The refusal is returned rather than thrown, so it can be graded + * on its wording. */ inspect( seatId: string, options?: { omitOrg?: boolean } ): Promise<{ facts?: Record; error?: string }>; - /** Has every fan-out request this mailbox started reached a terminal status? */ - fanOutSettled(mailboxId?: string): Promise; + /** + * Has every request on a coordinator's conversation, and on each delegate + * session it opened, reached a terminal status? False before the first post. + */ + settled(coordinatorId?: string): Promise; dispose(): Promise; } -/** The fields `openMailboxes` passes to `createSession`. */ -type CreateSessionOptions = { - flowKind: string; - userId: string; - sessionId?: string; - description?: string; - state?: Record; -}; +/** Text of a message item's content parts. */ +const textOf = (item: Record): string => + ((item.content as Array<{ text?: string }> | undefined) ?? []).map((part) => part.text ?? "").join(""); /** - * Read the tree, build the kinds, hire, and open the mailboxes. + * Read the tree, build the flows, hire, and open the operator's conversations. * * @param options The stores, the answer slot, the model resolver, and any controls. * @returns The live lab. Call `dispose()` when done. - * @throws If the tree does not load, or if any record refuses at the mint — which - * is BR-18, and deliberately fatal: nothing is hired and nothing is registered. + * @throws If the tree does not load, if the installation refuses a record (a + * coordinator naming a delegate no file declares — BR-10), or if any record + * refuses at the hire — BR-18, and deliberately fatal: nothing is hired and + * nothing is registered. */ export async function openLab(options: OpenLabOptions): Promise { const roster = await loadTree(options.root ?? LAB_TREE); @@ -297,16 +344,31 @@ export async function openLab(options: OpenLabOptions): Promise { const scopeCheck = options.scopeCheck ?? createScopeCheck(); - // The control mutates the RECORD, before the mint, so a redirected seat - // genuinely runs on the other document rather than being graded as if it did. + // Every control mutates the RECORD, before the installation is built, so a + // redirected seat genuinely runs on the other document rather than being + // graded as if it did. const overrides = options.documentOverrides ?? {}; const toolOverrides = options.toolOverrides ?? {}; - const workers = roster.workers.map((worker) => { + const noPost = new Set(options.noPostFor ?? []); + const treeCoordinatorId = roster.workers.filter(isCoordinator).map((w) => w.id).sort()[0]; + const workers: WorkerManifest[] = [...roster.workers, ...(options.extraWorkers ?? [])].map((worker) => { const declared = { ...worker.declared }; if (Object.hasOwn(overrides, worker.id)) declared.document = overrides[worker.id]; if (Object.hasOwn(toolOverrides, worker.id)) declared.tools = toolOverrides[worker.id]; - const redirected = { ...worker, declared }; - return options.mutateSkills === undefined + if (noPost.has(worker.id)) declared.flow = DESK_KIND; + let body = worker.body; + if (worker.id === treeCoordinatorId) { + if (options.alsoDelegate !== undefined) { + declared.delegates = [...((declared.delegates as string[] | undefined) ?? []), ...options.alsoDelegate]; + } + if (options.coordinatorRounds !== undefined) declared.rounds = options.coordinatorRounds; + // The charter is the record's BODY, so the control drops the body: the + // installation still builds and the post still routes; only the charter + // is gone. + if (options.dropCharter === true) body = ""; + } + const redirected = { ...worker, declared, body }; + return options.mutateSkills === undefined || isCoordinator(redirected) ? redirected : { ...redirected, @@ -314,95 +376,57 @@ export async function openLab(options: OpenLabOptions): Promise { }; }); - // The installation the seat flow runs its workers on. It reads the flow when - // it first needs it, so the flow is built on it just below. + // The installation every worker flow runs its workers on. It reads the flows + // when it first needs them, so each is built on it just below. It refuses a + // coordinator whose `delegates:` names a worker no file declares (BR-10). let workerFlows: Record = {}; const installation = createWorkerInstallation({ standardWorkers: workers, workerFlows: () => workerFlows as never, }); - const seatKind = defineProbeWorkerFlow({ + const probeOptions = { installation, answer: options.answer ?? serverAnswer, catalog: { [scopeCheck.key]: scopeCheck.block }, resources, ...(options.defaultModel === undefined ? {} : { defaultModel: options.defaultModel }), - ...(options.suppressPostBack === true ? { postBackEnabled: false } : {}), - }); - workerFlows = { [SEAT_KIND]: seatKind }; + }; + const seatKind = defineProbeWorkerFlow(probeOptions); + workerFlows = { + [SEAT_KIND]: seatKind, + // Only the flows a delegate takes posts on are the coordinator's to deliver to. + [COORDINATOR_KIND]: defineCoordinatorFlow({ installation, delegateFlows: [seatKind], routeModel: ROUTE_MODEL }), + ...(noPost.size > 0 ? { [DESK_KIND]: defineProbeWorkerFlow({ ...probeOptions, kind: DESK_KIND }) } : {}), + }; // Refuses the WHOLE roster when any worker would be refused on its turn, // naming the worker. Nothing is registered partially, so a refusal cannot // leave a short roster running. const hired = hireWorkforce(installation); - const seatCopy = hired.find((copy) => copy.kind === SEAT_KIND)!; - // Each worker's flow copy, by the worker's id. + const copyOf = (kind: string) => hired.find((copy) => copy.kind === kind)!; + const coordinatorCopy = copyOf(COORDINATOR_KIND); + // Each seat's flow copy, by the worker's id: every worker on a lab kind. const seats: Record = Object.fromEntries( - workers.map((worker) => [worker.id, seatCopy]) - ); - - // Who the router can reach. Every declared member of every mailbox that is - // also a hired seat, minus the ones a control withholds, plus any a control - // adds. A member left out is BR-9; a member that is no hired seat at all - // never reaches this map, which is BR-10. - // The control mutates the mailbox RECORD, so the extra member is genuinely - // declared and the framework's own member walk hands the router a delivery - // for it — rather than the lab pretending it did. - const treeMailboxes = roster.mailboxes.map((mailbox, index) => { - if (index !== 0) return mailbox; - const mutated = { ...mailbox }; - if (options.alsoMember !== undefined) { - mutated.declared = { - ...mutated.declared, - members: [ - ...(Array.isArray(mutated.declared.members) - ? (mutated.declared.members as string[]) - : []), - ...options.alsoMember, - ], - }; - } - // The charter is the record's BODY, so the control drops the body — the - // roster still binds, the mailbox still opens, and only the charter is - // gone. - if (options.dropCharter === true) mutated.body = ""; - return mutated; - }); - - const allMailboxes = [...treeMailboxes, ...(options.extraMailboxes ?? [])]; - const declaredMembers = new Set( - allMailboxes.flatMap((mailbox) => - Array.isArray(mailbox.declared.members) ? (mailbox.declared.members as string[]) : [] - ) + workers.filter((worker) => !isCoordinator(worker)).map((worker) => [worker.id, copyOf(String(worker.declared.flow))]), ); - - const withheld = new Set(options.omitAddressFor ?? []); - const addresses: Record = {}; - for (const member of [...declaredMembers].sort()) { - if (withheld.has(member)) continue; - if (!Object.hasOwn(seats, member)) continue; - addresses[member] = member; - } - - const notifyLog = createNotifyLog(); - const mailboxKind = defineMailboxFlow({ - notify: labNotify({ - addresses, - flowOf: () => seatCopy.id, - log: notifyLog, - ...(options.routeAuthoredLines === true ? { routeAuthoredLines: true } : {}), - }), - ...(options.inventory === true ? { inventory: true } : {}), - }); - - const registeredMailboxes = mailboxInstances(allMailboxes, { - kinds: { [MAILBOX_KIND]: mailboxKind as never }, - }); + const coordinatorIds = workers.filter(isCoordinator).map((worker) => worker.id).sort(); + + const writerFlow = + options.inventory === true + ? (() => { + const writer = inventoryWriterActions(INVENTORY_WRITER_KIND); + return defineFlow({ + kind: INVENTORY_WRITER_KIND, + // The seat write takes caller-supplied rows, so it is internal only. + internal: { actions: { ...writer } }, + } as never) as unknown as FlowInstance; + })() + : undefined; const state = createFlowState({ flows: Object.fromEntries([ - ...registeredMailboxes.map((instance) => [instance.id, instance]), ...hired.map((copy) => [copy.id, copy]), + ...(writerFlow === undefined ? [] : [[INVENTORY_WRITER_KIND, writerFlow]]), ]), stores: { default: { primary: options.stores } }, // Ignores the request. One caller, and a body read would be caller-controlled identity (BP-031). @@ -419,9 +443,10 @@ export async function openLab(options: OpenLabOptions): Promise { const call = async ( method: "GET" | "POST" | "DELETE", segments: string[], - body?: unknown + body?: unknown, + query = "" ): Promise<{ status: number; json: any }> => { - const request = new Request(`http://lab/api/flows/${segments.join("/")}`, { + const request = new Request(`http://lab/api/flows/${segments.join("/")}${query}`, { method, ...(body === undefined ? {} : { body: JSON.stringify(body) }), }); @@ -430,65 +455,52 @@ export async function openLab(options: OpenLabOptions): Promise { return { status: response.status, json: text.length > 0 ? JSON.parse(text) : undefined }; }; - const client = { - createSession: async (create: CreateSessionOptions): Promise => { - const { status, json } = await call("POST", [create.flowKind, "sessions"], { - userId: create.userId, - ...(create.sessionId === undefined ? {} : { sessionId: create.sessionId }), - ...(create.description === undefined ? {} : { description: create.description }), - ...(create.state === undefined ? {} : { state: create.state }), - }); - if (status >= 400) { - throw Object.assign(new Error(`createSession ${status}: ${JSON.stringify(json)}`), { status }); - } - return json; - }, - getSession: async (sessionId: string) => { - const { status, json } = await call("GET", ["sessions", sessionId]); - if (status >= 400) { - throw Object.assign(new Error(`getSession ${status}: ${JSON.stringify(json)}`), { status }); - } - const session = json.session ?? {}; - return { - flowKind: session.flowKind, - flowId: session.flowId, - userId: session.userId, - state: session.state, - }; - }, - deleteSession: async (sessionId: string): Promise => { - const { status, json } = await call("DELETE", ["sessions", sessionId]); - if (status >= 400) { - throw Object.assign(new Error(`deleteSession ${status}: ${JSON.stringify(json)}`), { status }); - } - }, + /** + * The operator's conversation with one coordinator, found by the worker it + * names or started naming it — through the real session routes, the way an + * app's `ensureWorkerSession` does. Found first, so a host rebuilt over the + * same store reads the conversation the first one wrote. + */ + const openConversation = async (coordinatorId: string): Promise => { + const query = `?${new URLSearchParams({ flowId: COORDINATOR_KIND, userId: LAB_USER_ID, "state.workerId": coordinatorId })}`; + const listed = await call("GET", ["sessions"], undefined, query); + if (listed.status >= 400) throw new Error(`list conversations ${listed.status}: ${JSON.stringify(listed.json)}`); + const found = (listed.json?.sessions ?? []) as Array<{ id: string }>; + if (found.length > 0) return found[0]!.id; + const created = await call("POST", [COORDINATOR_KIND, "sessions"], { + userId: LAB_USER_ID, + state: { workerId: coordinatorId }, + }); + if (created.status >= 400) throw new Error(`open a conversation with ${coordinatorId} ${created.status}: ${JSON.stringify(created.json)}`); + return String(created.json?.session?.id ?? created.json?.id); }; - await openMailboxes(allMailboxes, { client, userId: LAB_USER_ID }); - - const mailboxIds = allMailboxes.map((mailbox) => mailbox.id).sort(); - const treeMailboxId = treeMailboxes[0]!.id; - const mailboxInstance = registeredMailboxes.find((instance) => instance.kind === MAILBOX_KIND)!; + const conversations = new Map(); + for (const id of coordinatorIds) conversations.set(id, await openConversation(id)); + const conversationOf = (coordinatorId?: string): string => { + const id = coordinatorId ?? treeCoordinatorId!; + const session = conversations.get(id); + if (session === undefined) throw new Error(`no coordinator "${id}" was hired`); + return session; + }; const runtime = await state.getRuntime(); // `createFlowState` builds its own `RuntimeConfig` and takes no logger - // option, and the fan-out's requests are started by the dispatch operation + // option, and the deliveries' requests are started by the dispatch operation // from that same resolved object rather than from anything a caller passes // per action. Setting it here is the one place that reaches both. if (options.logger !== undefined) { (runtime.runtimeConfig as { logger?: unknown }).logger = options.logger; } - // The inventory, in-process, under the lab's org, once the mailbox sessions - // it registers from exist. A problem fails the open, naming it. - if (options.inventory === true) { - const flows: Record = { [MAILBOX_KIND]: mailboxInstance, [seatCopy.id]: seatCopy }; + // The inventory, in-process, under the lab's org: one row per hired worker, + // written from the lab's own writer flow. A problem fails the open, naming it. + if (writerFlow !== undefined) { const run = async (request: InventoryActionRequest): Promise => { - const flow = flows[request.flowKind]; - if (flow === undefined) throw new Error(`no flow is served under "${request.flowKind}"`); + if (request.flowKind !== INVENTORY_WRITER_KIND) throw new Error(`no flow is served under "${request.flowKind}"`); const result = (await runAction({ - flow, + flow: writerFlow, actionName: request.action, input: request.input, userId: request.userId, @@ -502,8 +514,8 @@ export async function openLab(options: OpenLabOptions): Promise { return result; }; const opened = await openInventory( - { seats: inventorySeats(installation), mailboxes: allMailboxes }, - { run, seatWriter: { flowKind: MAILBOX_KIND }, userId: LAB_USER_ID, orgId: LAB_ORG_ID }, + { seats: inventorySeats(installation), mailboxes: [] }, + { run, seatWriter: { flowKind: INVENTORY_WRITER_KIND }, userId: LAB_USER_ID, orgId: LAB_ORG_ID }, ); if (opened.problems.length > 0) { await state.dispose(); @@ -517,8 +529,7 @@ export async function openLab(options: OpenLabOptions): Promise { * `runAction` rather than the HTTP route, for one reason: the route is * fire-and-forget (202 plus a request id) and the request record carries no * output, so reading an action's result off it is not a thing that works. - * This is the same entry the route dispatches into, and the one the - * framework's own mailbox tests and this spec's POC drive. + * This is the same entry the route dispatches into. */ const act = async ( flow: FlowInstance, @@ -543,8 +554,7 @@ export async function openLab(options: OpenLabOptions): Promise { runtimeConfig: { ...runtime.runtimeConfig }, // The cast is load-bearing: `omitOrg` makes `orgId` optional here and // `runAction` requires it, which is the contract this lab probes. It - // also means NOTHING in this literal is typechecked — the reason the - // stray `orgId` above had to be caught by reading rather than by tsc. + // also means NOTHING in this literal is typechecked. } as never)) as { output?: unknown; error?: unknown }; } catch (error) { // A refusal the substrate throws rather than returns — an org-binding @@ -555,54 +565,88 @@ export async function openLab(options: OpenLabOptions): Promise { } }; + /** A session's stored items of one type, through the session's own state route. */ + const itemsOf = async (sessionId: string, type: string): Promise>> => { + const query = `?${new URLSearchParams({ include_items: "true", item_types: type, limit: "1000" })}`; + const read = await call("GET", ["sessions", sessionId, "state"], undefined, query); + if (read.status >= 400) throw new Error(`read ${sessionId}: ${read.status} ${JSON.stringify(read.json)}`); + return (read.json?.items ?? []) as Array>; + }; + + /** The sessions a conversation opened beneath it: its delegates' sessions, by their session ids. */ + const childrenOf = async (sessionId: string): Promise> => { + const read = await call("GET", ["sessions", sessionId, "children"]); + if (read.status >= 400) throw new Error(`read ${sessionId}'s children: ${read.status} ${JSON.stringify(read.json)}`); + return (read.json?.children ?? []) as Array<{ id: string }>; + }; + return { state, roster: { ...roster, workers }, seats, - mailboxIds, - mailboxId: treeMailboxId, - notifyLog, + coordinatorIds, + coordinatorId: treeCoordinatorId!, scopeCheck, - post: async (body: string, author?: string, mailboxId?: string) => { + coordinatorWorker: (id?: string) => { + const worker = installation.standardWorker(id ?? treeCoordinatorId!); + if (worker === undefined) throw new Error(`no coordinator "${id ?? treeCoordinatorId}" was hired`); + return { + delegates: (worker.declared.delegates as string[] | undefined) ?? [], + instructions: worker.body ?? "", + description: typeof worker.declared.description === "string" ? worker.declared.description : null, + }; + }, + + post: async (body: string, author?: string, coordinatorId?: string) => { const result = await act( - mailboxInstance, - "post", - { body, ...(author === undefined ? {} : { author }) }, - mailboxId ?? treeMailboxId + coordinatorCopy, + "run", + { message: body, ...(author === undefined ? {} : { author }) }, + conversationOf(coordinatorId) ); return { error: result.error === undefined ? undefined : messageOf(result.error) }; }, - mailboxState: async (mailboxId?: string) => { - const state = (await client.getSession(mailboxId ?? treeMailboxId)).state ?? {}; - return { - members: (state.members as string[]) ?? [], - instructions: (state.instructions as string) ?? "", - transcript: (state.transcript as TranscriptLine[]) ?? [], - }; + readConversation: async (coordinatorId?: string) => { + const session = conversationOf(coordinatorId); + const listed = await act(coordinatorCopy, LIST_DELEGATES, {}, session); + if (listed.error !== undefined) throw new Error(`read ${session}'s delegates: ${messageOf(listed.error)}`); + const delegates = ((listed.output as { delegates?: Array<{ worker: string }> } | undefined)?.delegates ?? []).map( + (record) => record.worker + ); + const lines = (await itemsOf(session, "message")).map((item) => ({ + id: String(item.id), + role: String(item.role), + author: typeof item.agentName === "string" ? item.agentName : null, + body: textOf(item), + item, + })); + const routes = (await itemsOf(session, "component")) + .filter((item) => item.component === COORDINATOR_ROUTE) + .map((item) => item.data as CoordinatorRouteRecord); + return { delegates, lines, routes }; }, - readMailbox: async (mailboxId?: string) => { - const result = await act(mailboxInstance, "read", {}, mailboxId ?? treeMailboxId); - if (result.error !== undefined) { - throw new Error(`read ${mailboxId ?? treeMailboxId}: ${messageOf(result.error)}`); + delegateSessions: async (coordinatorId?: string) => { + const workers: string[] = []; + for (const child of await childrenOf(conversationOf(coordinatorId))) { + const read = await call("GET", ["sessions", child.id]); + const worker = read.json?.session?.state?.workerId; + if (typeof worker === "string") workers.push(worker); } - const output = (result.output ?? {}) as { members?: string[]; transcript?: unknown[] }; - return { - members: output.members ?? [], - transcript: (output.transcript ?? []) as TranscriptLine[], - }; + return workers.sort(); }, inspect: async (seatId: string, inspectOptions?: { omitOrg?: boolean }) => { const seat = seats[seatId]; if (seat === undefined) throw new Error(`no seat "${seatId}" was hired`); - // A direct read: not a wakening by the fan-out, and it writes nothing, so - // the silent seat can be both silent (BR-13) and evidenced (BR-3). - // `omitOrg` is the same call with no org. The message does not say - // OrgRequired, so that probe prefixes the error's name for the grade. + // A direct read: not a delivery from the coordinator, and it writes + // nothing to any conversation, so the silent seat can be both silent + // (BR-13) and evidenced (BR-3). `omitOrg` is the same call with no org. + // The message does not say OrgRequired, so that probe prefixes the + // error's name for the grade. const omitOrg = inspectOptions?.omitOrg === true; const sessionId = `${omitOrg ? "no-org" : "inspect"}-${seatId}`; // The worker's session, created naming it, the way the session route does. @@ -641,35 +685,22 @@ export async function openLab(options: OpenLabOptions): Promise { }; }, - fanOutSettled: async (mailboxId?: string) => { - const requests = await runtime.stores.request.list({ - sessionId: mailboxId ?? treeMailboxId, - }); - const fanOuts = requests.filter((request: any) => request.actionName === "onPosted"); - return fanOuts.length > 0 && fanOuts.every((request: any) => request.status !== "in_progress"); + settled: async (coordinatorId?: string) => { + const conversation = conversationOf(coordinatorId); + const sessions = [conversation, ...(await childrenOf(conversation)).map((child) => child.id)]; + let any = false; + for (const sessionId of sessions) { + const requests = await runtime.stores.request.list({ sessionId }); + if (requests.length > 0) any = true; + if (requests.some((request: { status: string }) => request.status === "in_progress")) return false; + } + return any; }, dispose: () => state.dispose(), }; } -/** - * One line of a mailbox transcript, whole. - * - * Every field the store holds, not just the two a reader looks at: BR-15 - * compares complete records across a restart, and a type that omitted `id`, - * `at` or `principal` would quietly narrow that comparison to the fields it - * happened to name. - */ -export interface TranscriptLine { - id: string; - at: number; - principal: string; - author?: string; - authorVerified: boolean; - body: string; -} - /** One wording for whatever a refusal turns out to be. */ function messageOf(error: unknown): string { if (error instanceof Error) return error.message; diff --git a/goals/pentest-lab/lab/notify.mts b/goals/pentest-lab/lab/notify.mts deleted file mode 100644 index 47f98ca423..0000000000 --- a/goals/pentest-lab/lab/notify.mts +++ /dev/null @@ -1,184 +0,0 @@ -/** - * The lab's notify block — the `notify` slot `defineMailboxFlow` takes. - * - * **Not a second fan-out.** `mailbox-flow.ts` keeps the member walk: it reads - * the roster off the mailbox's own session state and runs this block once per - * declared member per post. What is the lab's here is only the *policy* — which - * posts route at all, and what address each member resolves to — because the - * dispatch seam refuses a target taken out of stored data, so a notify block - * has to declare its own. - * - * Two rules, and they are the whole of it: - * - * 1. **A post with an `author` routes to nobody.** A seat's own answer is a - * post like any other, and `mailbox-flow.ts` taps the hand-off for *every* - * post and passes `author` straight through. Without this gate two seats - * answer each other forever — 1 → 2 → 4 → 8 detached dispatches with - * nothing to stop them. The cycle break is the app's to write, which is - * why it is written here, in the open, rather than assumed. - * 2. **A member with no declared address is recorded and skipped.** The other - * members still run, and membership is never changed by a delivery. - * - * The addresses are a static id → dispatcher map, built once. It has to be - * static: `flowKind` on a dispatcher is an instance id, not a function, so - * "look the member up and dispatch to it" is not a shape this seam offers. - */ - -import { handler, sequencer, dispatcher } from "@flow-state-dev/core"; -import { mailboxNotifyInputSchema, type MailboxNotifyInput } from "@flow-state-dev/workforce"; -import { WORKER_ID_STATE_KEY } from "@flow-state-dev/workforce/browser"; -import { z } from "zod"; -import { BRIEF_ENTRY } from "./workforce/flows/workers/probe.mts"; - -/** What the router decided about one member's delivery, and why. */ -const decisionSchema = z.object({ - /** The member to dispatch to, or null when this delivery routes to nobody. */ - target: z.string().nullable(), - member: z.string(), - mailboxId: z.string(), - body: z.string(), -}); - -type Decision = z.infer; - -/** What the router recorded about one run — the evidence the gate grades BR-8, BR-9 and BR-13 on. */ -export interface NotifyLog { - /** - * Every member the fan-out actually dispatched to, in order. - * - * Recorded because a stray dispatch whose seat produces no post is otherwise - * indistinguishable from no dispatch at all — so BR-13's "the fan-out never - * woke `audit.scribe`" would have no evidence behind it, only an absence. - */ - addressed: string[]; - /** Every member the router saw with no declared address (BR-9). */ - skipped: string[]; - /** Every member whose line was refused routing because it carried an author (BR-8). */ - authored: string[]; - /** Every delivery the substrate refused, by message. */ - refusals: string[]; -} - -/** A fresh, empty log. */ -export function createNotifyLog(): NotifyLog { - return { addressed: [], skipped: [], authored: [], refusals: [] }; -} - -export interface LabNotifyOptions { - /** - * Member id → the worker it is delivered to. - * - * A member absent from this map is BR-9's case: recorded and skipped. Every - * id here is a standard worker the tree declares. - */ - addresses: Record; - /** The flow a worker runs on, where its delivery's session is opened. */ - flowOf: (workerId: string) => string; - /** Where the run's routing decisions are recorded. */ - log: NotifyLog; - /** - * Route a post that already carries an `author`. Default false. - * - * **V2's control, and nothing else.** Turning it on makes the router do the - * thing BR-8 forbids, so the gate can watch the forbidden dispatch happen - * rather than assert about an absence. It is only survivable with the seats' - * post-backs suppressed, which is the other half of that control. - */ - routeAuthoredLines?: boolean; -} - -/** - * Build the notify slot. - * - * @param options The address map, the log to record into, and V2's control switch. - * @returns The block to pass as `defineMailboxFlow({ notify })`. - */ -export function labNotify(options: LabNotifyOptions) { - const { addresses, log, flowOf } = options; - const members = Object.keys(addresses); - - /** - * The one decision point. Every condition below is a pure lookup on what this - * block returned, so the policy lives in exactly one place — a second copy of - * it in the conditions is how a router comes to route two different ways. - */ - const decide = handler({ - name: "lab-notify-decide", - inputSchema: mailboxNotifyInputSchema, - outputSchema: decisionSchema, - execute: (input: MailboxNotifyInput): Decision => { - const nowhere: Decision = { - target: null, - member: input.member, - mailboxId: input.mailboxId, - body: input.body, - }; - - // Rule 1 — the cycle break. - if (input.author !== undefined && options.routeAuthoredLines !== true) { - log.authored.push(input.member); - return nowhere; - } - - // Rule 2 — a declared member this lab has no address for. - if (!Object.hasOwn(addresses, input.member)) { - log.skipped.push(input.member); - return nowhere; - } - - log.addressed.push(input.member); - return { ...nowhere, target: input.member }; - }, - }); - - /** Captures why a delivery failed, at the point it fails. */ - const recordRefusal = handler({ - name: "lab-notify-refused", - inputSchema: z.unknown(), - outputSchema: z.object({ delivered: z.literal(false) }), - execute: (error: unknown) => { - log.refusals.push(error instanceof Error ? error.message : String(error)); - return { delivered: false as const }; - }, - }); - - /** - * One dispatcher per member, because `flowKind` is a static flow id. - * - * `session: { key, state }`, never `{ id }`: a seat has no session until - * something wakes it, and an exact id nothing created is refused - * `session-not-found` by name. The `key` child is derived, created on first - * delivery naming the member's worker, and inherits the sender's org — which - * is what carries the seats' document reads. - * Settled by a run, with the `{ id }` shape as its control. - */ - const toSeat = (member: string) => - dispatcher({ - name: `lab-notify-${member.replace(/\./g, "-")}`, - action: BRIEF_ENTRY, - flowKind: flowOf(addresses[member]!), - inputSchema: decisionSchema, - payload: (decision: Decision) => ({ - mailboxId: decision.mailboxId, - body: decision.body, - member: decision.member, - }), - session: { key: () => member, state: { [WORKER_ID_STATE_KEY]: addresses[member]! } }, - } as never); - - let seq: any = sequencer({ - name: "lab-notify", - inputSchema: mailboxNotifyInputSchema, - }).step(decide); - - for (const member of members) { - seq = seq.stepIf( - (decision: Decision) => decision.target === member, - // One member's refusal is absorbed here as well as by the framework's own - // rescue, so the reason is recorded rather than only counted. - (toSeat(member) as any).rescue([{ block: recordRefusal }]) - ); - } - - return seq; -} diff --git a/goals/pentest-lab/lab/refusal-trees/unknown-kind-corrected/teams/pentest/mailboxes/findings/MAILBOX.md b/goals/pentest-lab/lab/refusal-trees/unknown-kind-corrected/teams/pentest/workers/findings/WORKER.md similarity index 51% rename from goals/pentest-lab/lab/refusal-trees/unknown-kind-corrected/teams/pentest/mailboxes/findings/MAILBOX.md rename to goals/pentest-lab/lab/refusal-trees/unknown-kind-corrected/teams/pentest/workers/findings/WORKER.md index e1f57614b7..f352d059b0 100644 --- a/goals/pentest-lab/lab/refusal-trees/unknown-kind-corrected/teams/pentest/mailboxes/findings/MAILBOX.md +++ b/goals/pentest-lab/lab/refusal-trees/unknown-kind-corrected/teams/pentest/workers/findings/WORKER.md @@ -1,6 +1,8 @@ --- description: Where this tree reports what it found. -members: [pentest.recon, pentest.triage] +flow: coordinator +delegates: [pentest.recon, pentest.triage] +routing: everyone --- One finding per line. diff --git a/goals/pentest-lab/lab/refusal-trees/unknown-kind/teams/pentest/mailboxes/findings/MAILBOX.md b/goals/pentest-lab/lab/refusal-trees/unknown-kind/teams/pentest/workers/findings/WORKER.md similarity index 51% rename from goals/pentest-lab/lab/refusal-trees/unknown-kind/teams/pentest/mailboxes/findings/MAILBOX.md rename to goals/pentest-lab/lab/refusal-trees/unknown-kind/teams/pentest/workers/findings/WORKER.md index e1f57614b7..f352d059b0 100644 --- a/goals/pentest-lab/lab/refusal-trees/unknown-kind/teams/pentest/mailboxes/findings/MAILBOX.md +++ b/goals/pentest-lab/lab/refusal-trees/unknown-kind/teams/pentest/workers/findings/WORKER.md @@ -1,6 +1,8 @@ --- description: Where this tree reports what it found. -members: [pentest.recon, pentest.triage] +flow: coordinator +delegates: [pentest.recon, pentest.triage] +routing: everyone --- One finding per line. diff --git a/goals/pentest-lab/lab/refusal-trees/unknown-tool-corrected/teams/pentest/mailboxes/findings/MAILBOX.md b/goals/pentest-lab/lab/refusal-trees/unknown-tool-corrected/teams/pentest/workers/findings/WORKER.md similarity index 51% rename from goals/pentest-lab/lab/refusal-trees/unknown-tool-corrected/teams/pentest/mailboxes/findings/MAILBOX.md rename to goals/pentest-lab/lab/refusal-trees/unknown-tool-corrected/teams/pentest/workers/findings/WORKER.md index e1f57614b7..f352d059b0 100644 --- a/goals/pentest-lab/lab/refusal-trees/unknown-tool-corrected/teams/pentest/mailboxes/findings/MAILBOX.md +++ b/goals/pentest-lab/lab/refusal-trees/unknown-tool-corrected/teams/pentest/workers/findings/WORKER.md @@ -1,6 +1,8 @@ --- description: Where this tree reports what it found. -members: [pentest.recon, pentest.triage] +flow: coordinator +delegates: [pentest.recon, pentest.triage] +routing: everyone --- One finding per line. diff --git a/goals/pentest-lab/lab/refusal-trees/unknown-tool/teams/pentest/mailboxes/findings/MAILBOX.md b/goals/pentest-lab/lab/refusal-trees/unknown-tool/teams/pentest/workers/findings/WORKER.md similarity index 51% rename from goals/pentest-lab/lab/refusal-trees/unknown-tool/teams/pentest/mailboxes/findings/MAILBOX.md rename to goals/pentest-lab/lab/refusal-trees/unknown-tool/teams/pentest/workers/findings/WORKER.md index e1f57614b7..f352d059b0 100644 --- a/goals/pentest-lab/lab/refusal-trees/unknown-tool/teams/pentest/mailboxes/findings/MAILBOX.md +++ b/goals/pentest-lab/lab/refusal-trees/unknown-tool/teams/pentest/workers/findings/WORKER.md @@ -1,6 +1,8 @@ --- description: Where this tree reports what it found. -members: [pentest.recon, pentest.triage] +flow: coordinator +delegates: [pentest.recon, pentest.triage] +routing: everyone --- One finding per line. diff --git a/goals/pentest-lab/lab/scenario-trees/unknown-member/teams/pentest/mailboxes/briefing/MAILBOX.md b/goals/pentest-lab/lab/scenario-trees/unknown-member/teams/pentest/mailboxes/briefing/MAILBOX.md deleted file mode 100644 index 4f377d733a..0000000000 --- a/goals/pentest-lab/lab/scenario-trees/unknown-member/teams/pentest/mailboxes/briefing/MAILBOX.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -description: Declares a member that is no hired seat — BR-10's case. -members: [pentest.recon, pentest.nobody] ---- - -`pentest.nobody` is in no `workers/` folder anywhere. Membership is a declared -list, not a roster check, so this mailbox still opens. diff --git a/goals/pentest-lab/lab/scenario-trees/unknown-member/teams/pentest/workers/briefing/WORKER.md b/goals/pentest-lab/lab/scenario-trees/unknown-member/teams/pentest/workers/briefing/WORKER.md new file mode 100644 index 0000000000..257dcdbba2 --- /dev/null +++ b/goals/pentest-lab/lab/scenario-trees/unknown-member/teams/pentest/workers/briefing/WORKER.md @@ -0,0 +1,10 @@ +--- +description: Declares a delegate that is no worker — BR-10's case. +flow: coordinator +delegates: [pentest.recon, pentest.nobody] +routing: everyone +--- + +`pentest.nobody` is in no `workers/` folder anywhere. A coordinator's +delegates must each be a worker some file declares, so a host that hires +this coordinator is refused at load, naming it. diff --git a/goals/pentest-lab/lab/workforce/flows/workers/probe.mts b/goals/pentest-lab/lab/workforce/flows/workers/probe.mts index c86db9d77d..13903dd7fc 100644 --- a/goals/pentest-lab/lab/workforce/flows/workers/probe.mts +++ b/goals/pentest-lab/lab/workforce/flows/workers/probe.mts @@ -2,42 +2,44 @@ * The `probe` worker kind — one file under `workforce/flows/workers/`, * basename = kind id. * - * Custom kinds are authored here, file-per-kind. The host hands this factory - * to `hireWorkforce`'s `kinds` map. + * Custom kinds are authored here, file-per-kind. The host registers this + * flow on its worker installation, and the coordinator delivers posts to it. * - * **Not the built-in `agent` kind, and not by preference.** A mailbox's fan-out - * reaches a seat through an `internal` dispatch, which resolves - * `flow.internal.actions[action]` and never falls through to a public one; the - * built-in kind declares `actions.run` and no internal map at all, so a - * delivery into it refuses `no-entry`. That is settled by a run, not a read — - * see DECISIONS → Settled. What this kind proves is therefore the *convention* - * surface (a file's body, its skills, its document, its `tools:` fence) on a - * kind the lab wrote, and deliberately not the built-in kind under fan-out, - * because no such path exists. + * **Not the built-in `agent` kind, on purpose.** What this kind proves is the + * *convention* surface (a file's body, its skills, its document, its `tools:` + * fence) on a kind the lab wrote: a convention only holds if a second kind can + * implement it from the same description. A coordinator reaches a seat + * through an `internal` dispatch on {@link DELEGATED_POST_ENTRY}, which this + * kind declares with Workforce's `delegatedPostEntry`, the way the built-in + * kind does: the turn runs on the post, and its reply is handed back to the + * conversation that delivered it. * * The answering block is a **slot**, which is what lets the model-free gate and * the model-backed honesty check drive the same tree, the same hire and the * same wiring, differing by one block and nothing else (D2). * * The `tools:` fence is re-implemented here rather than imported: it is - * `defineAgentWorkerFlow`'s, and copying its *shape* — refused at the mint, - * against the app's catalog, by name — is the point. A convention only holds if - * a second kind can implement it from the same description. + * `defineAgentWorkerFlow`'s, and copying its *shape* — refused at the hire, + * against the app's catalog, by name — is the point. */ -import { dispatcher, handler, sequencer, defineFlow } from "@flow-state-dev/core"; +import { handler, sequencer, defineFlow } from "@flow-state-dev/core"; import type { DeclaredResources } from "@flow-state-dev/core"; import type { BlockContext, BlockDefinition } from "@flow-state-dev/core/types"; -import { MAILBOX_KIND, MAILBOX_SEAT_POST_ACTION, WORKER_ID_STATE_KEY, workerConfigOf, type WorkerInstallation } from "@flow-state-dev/workforce"; +import { + DELEGATED_POST_ENTRY, + WORKER_ID_STATE_KEY, + delegatedPostEntry, + delegatedPostOnFinished, + workerConfigOf, + type WorkerInstallation, +} from "@flow-state-dev/workforce"; import { z } from "zod"; import { workerDoor } from "../../../../../lib/worker-door.mts"; /** The kind name every `WORKER.md` in the tree names in its `flow:` line. */ export const SEAT_KIND = "probe"; -/** The internal entry a mailbox delivery is addressed to. */ -export const BRIEF_ENTRY = "brief"; - /** The public entry the gate reads a seat's own configuration through. */ export const INSPECT_ENTRY = "inspect"; @@ -56,12 +58,8 @@ const SEAT_TOOLS_KEY = "seatTools"; /** The lab's own setting: which document this seat reads, by the ref the convention minted. */ const DOCUMENT_KEY = "document"; -/** What a mailbox delivery hands the seat. */ -export const briefInputSchema = z.object({ - mailboxId: z.string(), - body: z.string(), - member: z.string(), -}); +/** A turn's input: the post as the seat hears it, `", through : "`. */ +const turnInputSchema = z.object({ message: z.string() }); /** * One skill as it rides in the bag — `InitialSkill`'s shape, structurally. @@ -87,9 +85,7 @@ const seatSkillSchema = z export const seatFactsSchema = z.object({ /** The worker this session runs, off the session. */ member: z.string(), - /** The mailbox this delivery named, or null when read directly through `inspect`. */ - mailboxId: z.string().nullable(), - /** The operator's line, or null on a direct read. */ + /** The post as this seat heard it, or null on a direct read through `inspect`. */ post: z.string().nullable(), /** This seat's `WORKER.md` body, as `instructions`. */ instructions: z.string(), @@ -107,19 +103,6 @@ export const seatFactsSchema = z.object({ export type SeatFacts = z.infer; -/** What the compose step hands the post-back dispatcher. */ -const postBackSchema = z.object({ - mailboxId: z.string(), - body: z.string(), - author: z.string(), -}); - -/** The sequencer-scoped state that carries the delivery envelope past the answer slot. */ -const briefStateSchema = z.object({ - mailboxId: z.string().nullable().default(null), - post: z.string().nullable().default(null), -}); - /** The parts of the bag this file reads where the bag's type is erased. */ interface SeatConfig { instructions?: string; @@ -241,32 +224,19 @@ function settingsSchema(catalog: Record>, defa * as "the document was empty", which is exactly the silent pass this lab exists * to refuse. * - * One block behind two doors: the `brief` sequencer's first step, and the + * One block behind two doors: the delegated post's turn's first step, and the * public `inspect` action. The second is not a convenience — BR-13 requires - * `audit.scribe` to put nothing in the transcript, so the transcript *cannot* - * carry its skill union, and a rule with no evidence path is what this epic - * exists to refuse. A direct read is not a wakening by the fan-out, and it - * writes nothing to the transcript, which is what lets the silent seat be both - * silent and evidenced. + * `audit.scribe` to put nothing in the conversation, so the conversation + * *cannot* carry its skill union, and a rule with no evidence path is what + * this epic exists to refuse. A direct read is not a delivery from the + * coordinator, and it writes nothing to the conversation, which is what lets + * the silent seat be both silent and evidenced. */ const readOwnFacts = handler({ name: "probe-read-own-facts", - inputSchema: z.object({ - mailboxId: z.string().optional(), - body: z.string().optional(), - }), + inputSchema: z.object({ message: z.string().optional() }), outputSchema: seatFactsSchema, - execute: async ( - input: { mailboxId?: string; body?: string }, - ctx: BlockContext - ): Promise => { - // The delivery envelope, stashed before the answer slot replaces the - // sequencer's running value. `ctx.sequencer` is absent when this runs as - // the `inspect` action's own block, which is the direct-read door. - if (input.mailboxId !== undefined && ctx.sequencer?.patchState !== undefined) { - await ctx.sequencer.patchState({ mailboxId: input.mailboxId, post: input.body ?? null }); - } - + execute: async (input: { message?: string }, ctx: BlockContext): Promise => { const config = workerConfigOf(ctx) as unknown as SeatConfig; // A seat that named no document is refused by name here, which is where the @@ -293,8 +263,7 @@ const readOwnFacts = handler({ return { member: seatOf(ctx), - mailboxId: input.mailboxId ?? null, - post: input.body ?? null, + post: input.message ?? null, instructions: config.instructions ?? "", documentRef: config.document, document: document ?? "", @@ -305,52 +274,6 @@ const readOwnFacts = handler({ }, }); -/** - * Turn the answer slot's line into a post addressed to the mailbox that woke - * this seat. - * - * The mailbox id comes off the sequencer's own state rather than off the - * running value, because the answer slot's output is a bare string and a - * generator's cannot be anything else. The author is the session's worker — the - * seat's own id, which is also its declared membership name, so BR-7's post - * is made under an identity the seat reads off itself rather than one the - * delivery told it to claim. - */ -const composePost = handler({ - name: "probe-compose-post", - inputSchema: z.string(), - outputSchema: postBackSchema, - execute: (line: string, ctx: BlockContext) => { - const state = (ctx.sequencer?.state ?? {}) as { mailboxId?: string | null }; - return { - mailboxId: state.mailboxId ?? "", - body: line, - author: seatOf(ctx), - }; - }, -}); - -/** - * Post the answer back into the mailbox. - * - * `session: { id }`, not `{ key }`: the mailbox's session already exists — - * `openMailboxes` created it — and a key-derived child is hashed with the parent - * session and lineage, so it could never name a shared mailbox. The inverse - * holds on the way in, where a seat's session does *not* yet exist; both halves - * are settled by a run. - */ -const postBack = dispatcher({ - name: "probe-post-back", - action: MAILBOX_SEAT_POST_ACTION, - flowKind: MAILBOX_KIND, - inputSchema: postBackSchema, - payload: (input: z.infer) => ({ - body: input.body, - author: input.author, - }), - session: { id: (input: z.infer) => input.mailboxId }, -} as never); - export interface ProbeWorkerFlowOptions { /** The installation that runs this flow's workers. */ installation: WorkerInstallation; @@ -368,14 +291,16 @@ export interface ProbeWorkerFlowOptions { /** The model a seat that names none runs on. */ defaultModel?: string; /** - * Whether an answering seat posts back into the mailbox. Default true. - * - * Turned off only by V2's control, which must observe a forbidden authored - * line being dispatched *without* the post-backs that would otherwise make - * the run 1 → 2 → 4 → 8 detached dispatches with nothing to stop it. A - * control whose failure state is "the run never ends" has no failure state. + * The kind to register the flow under. Defaults to {@link SEAT_KIND}, the + * one the tree's files name. The gate's BR-9 case registers it again under + * another kind, which takes no posts. + */ + kind?: string; + /** + * Whether the flow declares {@link DELEGATED_POST_ENTRY}, so its workers can + * be delegates that take posts. Default true. */ - postBackEnabled?: boolean; + takesPosts?: boolean; } /** @@ -385,35 +310,29 @@ export interface ProbeWorkerFlowOptions { * @returns The flow `hireWorkforce` registers one copy of, for every worker on it. */ export function defineProbeWorkerFlow(options: ProbeWorkerFlowOptions) { + const kind = options.kind ?? SEAT_KIND; const catalog = options.catalog ?? {}; const settings = settingsSchema(catalog, options.defaultModel ?? "openai/gpt-5.4-mini"); - const answering = sequencer({ - name: "probe-brief", - inputSchema: briefInputSchema, - stateSchema: briefStateSchema, - }) + // The turn a delegated post runs: read what this seat can see of itself, + // then answer. Its reply is what the entry hands back to the conversation. + const turn = sequencer({ name: "probe-turn", inputSchema: turnInputSchema }) .step(readOwnFacts) .step(options.answer); - const brief = - options.postBackEnabled === false - ? answering - : answering.step(composePost).step(postBack); - const installation = options.installation; /** Loads the turn's worker, on every run of a request, so `workerConfigOf` reads it. */ const loadWorker = handler({ - name: `${SEAT_KIND}-load-worker`, + name: `${kind}-load-worker`, inputSchema: z.unknown(), resources: { ...installation.resources }, execute: async (_input: unknown, ctx: BlockContext) => ({ - worker: (await installation.resolveWorker(ctx as never, SEAT_KIND)).id, + worker: (await installation.resolveWorker(ctx as never, kind)).id, }), }); return defineFlow({ - kind: SEAT_KIND, + kind, // One copy for every worker on it; each session names its worker. cardinality: "collection", configSchema: settings, @@ -422,19 +341,19 @@ export function defineProbeWorkerFlow(options: ProbeWorkerFlowOptions) { // unconditional, so the org registry is always built for a request that // reaches the reading block. resources: { ...options.resources, ...installation.resources }, - request: { onStarted: loadWorker }, + // `onFinished` tells the coordinator when a delivered post's run was + // cancelled before it answered, as the built-in kind's does. + request: { onStarted: loadWorker, onFinished: delegatedPostOnFinished }, actions: { ...workerDoor, [INSPECT_ENTRY]: { block: readOwnFacts, description: "Read what this seat can see of its own configuration. Writes nothing.", }, }, - internal: { - actions: { - // Reachable only by an `internal` dispatch from a block running in one - // of this app's own requests — which is what a mailbox's fan-out is. - [BRIEF_ENTRY]: { block: brief, inputSchema: briefInputSchema }, - }, - }, + // Reachable only by an `internal` dispatch from a block running in one of + // this app's own requests — which is what a coordinator's delivery is. + ...(options.takesPosts === false + ? {} + : { internal: { actions: { [DELEGATED_POST_ENTRY]: delegatedPostEntry(turn) } } }), } as never); } diff --git a/goals/pentest-lab/lab/workforce/teams/audit/workers/scribe/WORKER.md b/goals/pentest-lab/lab/workforce/teams/audit/workers/scribe/WORKER.md index d8588cade9..749529d77f 100644 --- a/goals/pentest-lab/lab/workforce/teams/audit/workers/scribe/WORKER.md +++ b/goals/pentest-lab/lab/workforce/teams/audit/workers/scribe/WORKER.md @@ -6,6 +6,6 @@ document: rules-of-engagement You are the scribe seat on the audit team. -You are not a member of the pentest team's findings mailbox, so nothing there -reaches you. Answer in one short line carrying the exact token +You are not a delegate of the pentest team's findings coordinator, so nothing +posted there reaches you. Answer in one short line carrying the exact token WORKER-SCRIBE-B19F7. diff --git a/goals/pentest-lab/lab/workforce/teams/pentest/mailboxes/findings/MAILBOX.md b/goals/pentest-lab/lab/workforce/teams/pentest/workers/findings/WORKER.md similarity index 69% rename from goals/pentest-lab/lab/workforce/teams/pentest/mailboxes/findings/MAILBOX.md rename to goals/pentest-lab/lab/workforce/teams/pentest/workers/findings/WORKER.md index 3bea41657d..262bf6a525 100644 --- a/goals/pentest-lab/lab/workforce/teams/pentest/mailboxes/findings/MAILBOX.md +++ b/goals/pentest-lab/lab/workforce/teams/pentest/workers/findings/WORKER.md @@ -1,6 +1,8 @@ --- description: Where the pentest team reports what it found. -members: [pentest.recon, pentest.triage] +flow: coordinator +delegates: [pentest.recon, pentest.triage] +routing: everyone --- Report one finding per line. Name the rule you checked it against. diff --git a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md index 446f5baaed..7eeab09dda 100644 --- a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md +++ b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md @@ -40,3 +40,9 @@ The **import** and **source** legs have no control: a checkout without the expor | 2026-09-26 | FIX-1602 branch on a52bab37a, uncommitted | scripted | FAIL (by hand) | The export removed from the mailbox barrel: failed at **import** (`@flow-state-dev/workforce exports no wakeMemberSeats`). Restored. | | 2026-09-26 | FIX-1602 branch on a52bab37a, uncommitted | scripted | FAIL (by hand) | Kitchen-sink's `mailbox-notify.ts` and `workforce-shell.ts` put back as they were before the move: failed at **source** (`builds its own dispatcher`, `never calls wakeMemberSeats`, `SEAT_ASKS still carries a wake column`). Restored. | | 2026-09-26 | FIX-1602 branch on a52bab37a, uncommitted | scripted | **PASS** | First verdict. `desk.amy` and `desk.oz` each hold one conversation of `desk.front` with both posts heard once and answered (`u_fresh_host in desk.front: wake-token-a… can someone look at the refund queue?`); `desk.amy`'s own post heard by no seat; `desk.ned` holds nothing; source clean. Also PASS with every folder and id renamed (`help.lobby`, `help.kai`/`help.bo` agents, `help.lu` note). | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | FAIL (old check) | **Converted for FIX-1792 P1; the check before the conversion taken first.** The previous `run.mts` on the converted fixture (`desk.front` a coordinator `WORKER.md`): `TypeError` reading `declared` — the tree holds no mailbox for it to read. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted, kitchen-sink not yet converted | scripted | FAIL (by hand) | The converted check before kitchen-sink's own conversion: every leg green but **source**, whose kitchen-sink half found `workforce/hire.ts` registering no `defineCoordinatorFlow`. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Converted.** `defineCoordinatorFlow` resolves from the package; the host registers it and builds no dispatcher or router; `desk.front` routes to everyone; kitchen-sink registers `defineCoordinatorFlow` too. `desk.amy` and `desk.oz` each hold one conversation under the person's conversation with `desk.front`, each post heard once and answered, two answer lines each, and neither heard the other's answer; `desk.ned` (on `note`) holds no conversation. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-wake`: **woken** and **answers** only — `desk.amy` and `desk.oz` hold 0 conversations, and 0 answer lines. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=answers-go-on`: **answers** and **quiet** only — 4 answer lines each, and each heard the other's answer (*desk.oz, through desk.front: [reply:fresh-host] noted.*). | +| 2026-10-09 | same | scripted | FAIL (by hand) | `defineCoordinatorFlow` dropped from the package barrel: failed at **import** only. Restored. | diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md index b12ad0eb07..73b128b3e2 100644 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md @@ -59,3 +59,11 @@ The **import** and **source** legs have no control; each was reddened by hand, l | 2026-09-27 | FIX-1610 branch on 315976e5f, uncommitted | scripted | **PASS** | After the history-window fold: the route reads each post's lines and hold from the mailbox's route ledger, kept as each line is, not from the session's items. PASS, and each control again failed at exactly its own legs: `no-route` at one, lands, followup, held, context, fallback; `no-landing` at one, lands, followup, held, context; `no-transcript` at followup; `no-context` at context. `a-fresh-host-wakes-its-member-agents` still PASS. | | 2026-09-27 | FIX-1610 branch on ab3e0a400, uncommitted | scripted | **PASS** | After the stale-ledger fold: a ledger without the newest line in the request's window starts again from the window's lines. PASS, and each control again failed at exactly its own legs: `no-route` at one, lands, followup, held, context, fallback; `no-landing` at one, lands, followup, held, context; `no-transcript` at followup; `no-context` at context. `a-fresh-host-wakes-its-member-agents` still PASS. | | 2026-09-27 | FIX-1610 branch on 465ee1f45, uncommitted | scripted | **PASS** | After the ledger fold: a kind built with a route keeps every mailbox's lines in the ledger, routed or not, and a kind built without one drops a ledger at its first post. PASS, and each control again failed at exactly its own legs: `no-route` at one, lands, followup, held, context, fallback; `no-landing` at one, lands, followup, held, context; `no-transcript` at followup; `no-context` at context. `a-fresh-host-wakes-its-member-agents` still PASS. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | FAIL (old check) | **Converted for FIX-1792 P1; the check before the conversion taken first.** The previous `run.mts` on the converted fixture (`help` and `lounge` coordinator `WORKER.md` files): *the fixture needs one routed and one unrouted mailbox*. | +| 2026-10-09 | same | scripted | FAIL | **A faithful port first, every leg kept.** On the coordinator: **followup** — `tok-…p3` heard by `support.general`, not `support.accounts` (the one evaluation call is handed the post, not the line it follows); **context** — `support.general` answered *Buy what?*. Every other leg green. These two legs are retired above (BR-24); the product owner is asked on the P1 PR. | +| 2026-10-09 | same | scripted | **PASS** | **Converted.** `defineCoordinatorFlow` resolves; the host registers it and builds no dispatcher or router; one coordinator on `best-fit`. The device, account and framework posts each reached their specialist alone and landed once under its name; the unclear post landed once by the fallback; the post sent while `support.devices` had not answered was held for it with no evaluation; `support.lounge` delivered the one post to all four, each answer landing once. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-route`: **one**, **lands**, **held** and **fallback** only — every post heard by all four, four answer lines each. **everyone** green. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-landing`: **one**, **lands**, **fallback** and **everyone** only — no answer lands, so best fit's hold keeps every later post on `support.devices`. **held** green. | +| 2026-10-09 | same | scripted | FAIL (by hand) | The host given its own `dispatcher(…)`: failed at **source** only. Restored. | +| 2026-10-09 | same | route `vercel/typesafe-ai/jev`, answers `vercel/openai/gpt-5.4-mini` | FAIL (live, follow-up pinned) | `GOAL_LIVE=1` with the follow-up still pinned to the laptop's specialist: the follow-up went to `support.accounts`, not `support.devices`; *Where can I buy it?* went to `support.general`, which answered about something else. The same finding as the faithful port, on real models. | +| 2026-10-09 | same | as above | **PASS** (live) | `GOAL_LIVE=1`, the follow-up unpinned: each of the seven posts reached one delegate and landed one line under its name; the follow-up reached `support.accounts` again. | diff --git a/specs/issues/FIX-1792/poc/inventory/check.mjs b/specs/issues/FIX-1792/poc/inventory/check.mjs index 651235a883..767f817ff8 100644 --- a/specs/issues/FIX-1792/poc/inventory/check.mjs +++ b/specs/issues/FIX-1792/poc/inventory/check.mjs @@ -86,6 +86,7 @@ const REMOVED_EXPORTS = [ // S9 · best fit's mailbox wrapper, the wake, post-to-mailbox "routeByPurpose", "RouteByPurposeOptions", "wakeMemberSeats", "WakeMemberSeatsOptions", "hearingSeatsById", "reachableSeat", "mailboxPostCapability", "MAILBOX_POST_CAPABILITY", "POST_TO_MAILBOX_TOOL", "PostToMailboxInput", "postToMailboxInputSchema", + "workerMailboxPostCapability", // S9 · the inventory's mailbox and membership rows, devtool's and Shift Manager's readers of them "MailboxInventoryRow", "mailboxInventoryRowSchema", "defineMailboxInventoryCollection", "membershipKey", "MailboxRow", "MembershipRow", "toMailboxRow", "mailboxesBySeat", "mailboxOf", @@ -134,6 +135,7 @@ const KEPT_EXPORTS = { INVENTORY_REGISTER_SEATS: "the seats inventory stays", inventorySeatsRegisteredSchema: "the seats inventory stays", inventoryWriterActions: "the seats inventory stays", + workerIdOfTurn: "the agent flow's own; it moves out of mailbox-post-capability.ts when S9 removes that file", }; /** Where discovery's pinned domain list lives; `--after` requires it without `mailboxes` (S9). */ @@ -157,12 +159,17 @@ const SKIP = (f) => * file becomes a Storefront workstream its lead worker opens; no WORKER.md), * `coordinator` (no board, or a board that goes with its feature), `removed` * (goes with the goal or legs it served). No old file is kept: D2. + * + * `done` names the PR (or issue) that converted or removed the file. A done + * file must be gone from the tree, and the census still counts it, from its + * row: `boards`, and `had` for the other keys it carried. So EXPECT stays the + * spec's census, whichever PRs have landed. */ const FILES = { "apps/kitchen-sink/workforce/teams/support/mailboxes/help/MAILBOX.md": - { target: "coordinator", boards: ["escalations"], use: "kitchen-sink's help desk. Its escalations board goes with the escalation feature, removed by the product owner (2026-10-06)" }, + { done: "P1", had: { boardActions: 1, routing: 1 }, target: "coordinator", boards: ["escalations"], use: "kitchen-sink's help desk. Its escalations board goes with the escalation feature, removed by the product owner (2026-10-06)" }, "goals/mailbox-boards/it-hands-a-task-to-a-fresh-hire/fixtures/workforce/teams/ops/mailboxes/desk/MAILBOX.md": - { target: "session", boards: ["work"], use: "a coordinator hires a worker and files it a task by name" }, + { done: "FIX-1788, which retired its goal", target: "session", boards: ["work"], use: "a coordinator hires a worker and files it a task by name" }, "goals/mailbox-boards/it-runs-a-row-a-file-declared-board-holds/fixtures/workforce/teams/eng/mailboxes/feature/MAILBOX.md": { target: "session", boards: ["triage", "parked"], use: "one worker files a row and another's board runs it; the filer has the task tools through its delegates and files on its own session board; parked is drained by nobody and goes" }, "goals/mailbox-boards/it-runs-a-row-a-file-declared-board-holds/fixtures/workforce/teams/eng/mailboxes/notices/MAILBOX.md": @@ -174,17 +181,17 @@ const FILES = { "goals/manager-queue-lab/lab/workforce/teams/eng/mailboxes/queue/MAILBOX.md": { target: "session", boards: ["work"], use: "the manager files one row per desk; each desk's worker takes only its own" }, "goals/pentest-lab/lab/refusal-trees/unknown-kind-corrected/teams/pentest/mailboxes/findings/MAILBOX.md": - { target: "coordinator", boards: [], use: "a refusal tree's mailbox; the refusal is about a worker" }, + { done: "P1", target: "coordinator", boards: [], use: "a refusal tree's mailbox; the refusal is about a worker" }, "goals/pentest-lab/lab/refusal-trees/unknown-kind/teams/pentest/mailboxes/findings/MAILBOX.md": - { target: "coordinator", boards: [], use: "a refusal tree's mailbox; the refusal is about a worker" }, + { done: "P1", target: "coordinator", boards: [], use: "a refusal tree's mailbox; the refusal is about a worker" }, "goals/pentest-lab/lab/refusal-trees/unknown-tool-corrected/teams/pentest/mailboxes/findings/MAILBOX.md": - { target: "coordinator", boards: [], use: "a refusal tree's mailbox; the refusal is about a worker" }, + { done: "P1", target: "coordinator", boards: [], use: "a refusal tree's mailbox; the refusal is about a worker" }, "goals/pentest-lab/lab/refusal-trees/unknown-tool/teams/pentest/mailboxes/findings/MAILBOX.md": - { target: "coordinator", boards: [], use: "a refusal tree's mailbox; the refusal is about a worker" }, + { done: "P1", target: "coordinator", boards: [], use: "a refusal tree's mailbox; the refusal is about a worker" }, "goals/pentest-lab/lab/scenario-trees/unknown-member/teams/pentest/mailboxes/briefing/MAILBOX.md": - { target: "coordinator", boards: [], use: "names a member no file declares; it opened and skipped it, and is now refused at load" }, + { done: "P1", target: "coordinator", boards: [], use: "names a member no file declares; it opened and skipped it, and is now refused at load" }, "goals/pentest-lab/lab/workforce/teams/pentest/mailboxes/findings/MAILBOX.md": - { target: "coordinator", boards: [], use: "one post wakes both declared workers" }, + { done: "P1", target: "coordinator", boards: [], use: "one post wakes both declared workers" }, "goals/shift-manager/it-briefs-and-talks-with-the-chief-of-staff/lab-no-cos/workforce/teams/desk/mailboxes/front/MAILBOX.md": { target: "session", boards: ["work"], use: "the landing summary counts what waits and runs, in a lab with no chief of staff" }, "goals/shift-manager/it-briefs-and-talks-with-the-chief-of-staff/lab/workforce/teams/desk/mailboxes/front/MAILBOX.md": @@ -200,15 +207,15 @@ const FILES = { "goals/task-run-link/it-names-the-run-working-each-task/fixtures/workforce/teams/lab/mailboxes/desk/MAILBOX.md": { target: "session", boards: ["work"], use: "each handed-off row links the run working it" }, "goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/fixtures/workforce/teams/desk/mailboxes/front/MAILBOX.md": - { target: "coordinator", boards: [], use: "a post wakes each agent member once, never on its own post" }, + { done: "P1", target: "coordinator", boards: [], use: "a post wakes each agent member once, never on its own post" }, "goals/workforce-mailboxes/a-pre-rename-lab-is-refused-by-name/fixtures/tree/teams/desk/mailboxes/front/MAILBOX.md": { target: "removed", boards: [], use: "the pre-rename goal's renamed tree; it goes with the goal, whose refusal is removed (D2)" }, "goals/workforce-mailboxes/a-pre-rename-lab-is-refused-by-name/fixtures/tree/teams/desk/mailboxes/notices/MAILBOX.md": { target: "removed", boards: [], use: "the pre-rename goal's mailbox on a kind of its own; it goes with the goal (D2)" }, "goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/mailboxes/help/MAILBOX.md": - { target: "coordinator", boards: [], use: "a routed post gets one specialist's answer" }, + { done: "P1", had: { routing: 1 }, target: "coordinator", boards: [], use: "a routed post gets one specialist's answer" }, "goals/workforce-mailboxes/a-routed-post-gets-one-answer/fixtures/workforce/teams/support/mailboxes/lounge/MAILBOX.md": - { target: "coordinator", boards: [], use: "a mailbox without routing behaves as before" }, + { done: "P1", target: "coordinator", boards: [], use: "a mailbox without routing behaves as before" }, "packages/shift-manager/teams/devteam/workforce/teams/eng/mailboxes/feature/MAILBOX.md": { target: "workstream", boards: ["work"], use: "the DevTeam builds a feature; the storefront project claims it, and its coding runs find the project through the claim. A Storefront workstream the EM leads and files on" }, "packages/shift-manager/teams/devteam/workforce/teams/eng/mailboxes/triage/MAILBOX.md": @@ -259,6 +266,12 @@ function frontmatter(text) { * Each line is `path: "CLASS · surface · why"`; the surface IDs are PLAN.md's. */ const FILE_CLASS = { + "packages/workforce/test/reachable-seat.test.ts": "R · S9 · tests of reachableSeat, removed with the wake", + "packages/workforce/test/projects-harness.ts": "E · S9 · the harness registers the mailbox flow", + "packages/workforce/src/workers/register.ts": "E · S9 · the hire's mailboxBoards option goes", + "packages/workforce/src/best-fit.ts": "E · S9 · a comment names the mailbox's route wrapper, routeByPurpose", + "packages/shift-manager/test/devteam-cos-tool-names.test.ts": "E · S7 · the chief of staff loses setWorkstreams", + "packages/shift-manager/test/devteam-chief-of-staff-coordinator.test.ts": "C · S7 · the DevTeam host's mailboxes option in a test; the host moves off mailboxes", ".omp/extensions/mailbox.ts": "U · — · the agent-mailbox tool's command, not Workforce", "apps/docs/docs/client/react.md": "E · S12 · an example opens a session on flowKind \"mailbox\"", "apps/docs/docs/devtool/overview.md": "E · S12 · the inventory view's registered-mailboxes row", @@ -469,6 +482,7 @@ const FILE_CLASS = { */ const GOALS = { "goals/README.md": "EDIT · the word, FIX-1796", + "goals/lib/workers.mts": "EDIT · the hire's mailboxBoards option goes with S9", "goals/agent-discovery/an-orchestrator-routes-a-task-by-asking": "EDIT · a roster literal's mailboxes field", "goals/design-system/skins-reused-components-from-one-token-set": "EDIT · a rail label in a demo host", "goals/devforce-lab/it-codes-in-the-projects-repository": "CONVERT · the run finds its project through the feature workstream", @@ -568,10 +582,23 @@ function run(mode) { if (Array.isArray(fm.members) && fm.members.length === 0) census.emptyMembers += 1; const row = FILES[f]; if (!row) { problems.push(`unclassified MAILBOX.md: ${f}`); continue; } + if (row.done !== undefined) continue; byTarget[row.target] = (byTarget[row.target] ?? 0) + 1; if (row.boards.join(",") !== fm.boards.join(",")) problems.push(`boards differ for ${f}: table [${row.boards}] file [${fm.boards}]`); } - for (const f of Object.keys(FILES)) if (!mailboxes.includes(f) && mode !== "after") problems.push(`listed but not found: ${f}`); + for (const [f, row] of Object.entries(FILES)) { + if (row.done !== undefined) { + // Converted or removed: gone from the tree, and counted from its row. + if (mailboxes.includes(f)) problems.push(`marked done (${row.done}) but still a MAILBOX.md: ${f}`); + census.files += 1; + if (row.boards.length > 0) census.withBoards += 1; + census.boards += row.boards.length; + for (const [k, n] of Object.entries(row.had ?? {})) census[k] += n; + byTarget[row.target] = (byTarget[row.target] ?? 0) + 1; + continue; + } + if (!mailboxes.includes(f) && mode !== "after") problems.push(`listed but not found: ${f}`); + } // Part 2 const classed = {}; From b94a0e5c2b99dcf6cbcc373df25093d5b697bf6e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 01:26:54 +0000 Subject: [PATCH 03/14] docs(goals): a-person-talks rewritten on the coordinator, with its verdicts (FIX-1792 P1) The check's head states the rewrite: leg e and no-filing go with the escalation feature, main's control goes, and the smoke's follow-up is no longer pinned. The checker's chat-assistant row moves to S9 with its registry copy. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- .../goal.md | 49 +++++++++++-------- specs/issues/FIX-1792/poc/inventory/check.mjs | 2 +- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md index 404801f2cc..2fbd018998 100644 --- a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md +++ b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md @@ -1,39 +1,45 @@ # kitchen-sink-talk › a person talks to a seat, a mailbox, and back -**Issue:** FIX-1601, the closure of the epic FIX-1592. Legs a to c are the epic's goal (ER-17), the escalation is part 2 (ER-25), `seg` is part 4's segmentation row. The plan is `specs/issues/FIX-1601/PLAN.md`, which owns the leg definitions. +**Issue:** FIX-1601, the closure of the epic FIX-1592. Legs a to c are the epic's goal (ER-17), `seg` is part 4's segmentation row. The plan is `specs/issues/FIX-1601/PLAN.md`, which owns the leg definitions. Rewritten by FIX-1792 P1; the folder name is kept so its verdict history stays in one place. -**Outcome:** A person asks `support.help` a question and, without reloading, sees the one specialist whose purpose fits start work and then answer under its own name. Three questions reach three specialists, one each, and each specialist keeps only its own cases. A direct conversation with `support.devices` remembers its first message, stays out of the mailbox, and is still there after a reload. A case that needs a person is filed onto `escalations`, and the specialist says so. The gated legs run keyless; a real-model smoke gets one answer per post. +**Outcome:** A person asks `support.help`, the support desk's coordinator, a question and, without reloading, sees the one specialist whose purpose fits start work and then answer under its own name. Three questions reach three specialists, one each, and each specialist keeps only its own cases. A direct conversation with `support.devices` remembers its first message, stays out of the conversation with `support.help`, and is still there after a reload. The gated legs run keyless; a real-model smoke gets one answer per post. -**Input:** `fixtures/input.json`: the mailbox, its board and the person's label; the four specialists by role, the fallback and the specialist part 2 routes to; the scripted markers (`[route:]`, the text answer `[scenario:wake-after-a-hold]` and the post-tool answer `[scenario:reply-after-a-hold]`, each of which holds a few seconds before it answers so a working row has time to show, `[scenario:talk-to-seat]`, `[scenario:recall]`, `[scenario:needs-a-person]`) and the reply markers each writes; how long a line may take (15 s); the smoke's posts, which are FIX-1610's `poc/route-choice` posts plus the wifi line its follow-up leans on. `fixtures/today-main.json` is the same with today's `main` roster, for the `main` control. Held-out: every post carries a fresh token shaped `/[a-z]+-token-[a-z0-9]+/` (what the recall scenario names), and only this run's tokens are graded, so a line another run left behind can never be the one that passes. +**Rewritten (FIX-1792 P1).** `support.help` is a coordinator worker file on `routing: best-fit`, no longer a mailbox, and the escalation feature is removed, not replaced (FIX-1792 BR-14; product owner, 2026-10-06). What changed, and why: + +- **e** (part 2: a case filed onto `escalations`, `e:line`, `e:row-open`, `e:row`, `e:warning`) and the `no-filing` control go with the escalation feature. Nothing files a case. +- A specialist's posts are no longer its runs "in the mailbox": each is the specialist's own session for the person's conversation with `support.help` (a session on the `agent` flow, listed in the rail as a run of the conversation). **b:only-its-own** and **seg** read those. Since FIX-1788 the rail has no row per specialist, so which sessions are a specialist's is read from the server by the session's `workerId`, as an index; what each holds is read off the page. **c1** starts its conversation with "Talk" in the roster panel rather than "New conversation" on a rail row. +- `c2:mailbox` is renamed `c2:coordinator`: the conversation with `support.help` shows nothing of the direct talk. +- The `main` control and `fixtures/today-main.json` go: the app it ran (before FIX-1609 and FIX-1610) has no coordinator to post to, so this check cannot run there. +- `no-landing` now takes every answer away, not only a text answer that skipped the post tool: a delegate's answer is its turn's reply, handed back to the coordinator, and the control's flow hands nothing back. It names **b:line** beside **a:line**. +- The smoke's follow-up ("It sees it. It fails right after the password.") is no longer pinned to its antecedent's specialist, and its answer is not graded on naming the subject. Best fit's one evaluation is handed the post alone, not the lines before it, so a follow-up is routed on its own words. This is the same finding as `workforce-mailboxes/a-routed-post-gets-one-answer`'s retired **followup** and **context** legs, put to the product owner on FIX-1792's P1 PR. + +**Input:** `fixtures/input.json`: the coordinator and the person's label; the four specialists by role and the fallback; the scripted markers (`[route:]`, the text answer `[scenario:wake-after-a-hold]` and the post answer `[scenario:reply-after-a-hold]`, each of which holds a few seconds before it answers so a working row has time to show, `[scenario:talk-to-seat]`, `[scenario:recall]`) and the reply markers each writes; how long a line may take (15 s); the smoke's posts, which are FIX-1610's `poc/route-choice` posts plus the wifi line's follow-up. Held-out: every post carries a fresh token shaped `/[a-z]+-token-[a-z0-9]+/` (what the recall scenario names), and only this run's tokens are graded, so a line another run left behind can never be the one that passes. **Signal:** one real browser against kitchen-sink's **production build**, built once by the run, served by `next start` on the scripted model, the in-memory store and **no model key**. The page is loaded once per journey. Each leg reads the **open page** first and reloads once, at its end, and reads again. -- **a** (P1a). Post `[route:support.devices] [scenario:wake-after-a-hold]` and token A. Graded as **a:working**, the panel shows `support.devices is working` in a reading before the one that first shows its line; **a:line**, within 15 s of Send exactly one line answers the post, by `support.devices`, carrying `[reply:wake]`; **a:clears**, the row is gone within 5 s of that line; **a:alone**, no other seat shows as working; **a:kept**, after the reload the post and its one line are still there. The text answer never calls the post tool, so the line landed by FIX-1610's rule alone. -- **b** (P1b). Three posts carrying `[scenario:reply-after-a-hold]`, each after the previous answer shows and its row clears: B1 routed to `support.devices`, B2 to `support.accounts`, B3 with no route marker, for the fallback `support.general`. **b:working**, for each post the panel shows its seat working in a reading before the one that first shows its line; **b:line**, each gets exactly one line carrying its token and `[reply:in-mailbox]`, by its seat, within 15 s; **b:clears**, each post's row is gone within 5 s of its line (B3's too), and no row still shows when the next post would go; **b:alone**, no other seat shows as working, counting a row left from an earlier post; **b:only-its-own**, read from each specialist's conversation in the mailbox, on the open page and after the reload: `devices` holds A and B1, `accounts` B2, `general` B3, each heard once with a reply under it, `fsd` holds nothing, and nobody holds another's token; **b:kept**, after the reload each post and its line show once. -- **c1** (P1c). "New conversation" on `support.devices`, `[scenario:talk-to-seat]` and token C1: **c1:turn**, C1 is the person's turn, once; **c1:reply**, `[reply:talk-to-seat]` under it; on the open page and after the reload. -- **c2** (P1c). In that conversation, `[scenario:recall]` and token C2: **c2:turn**, C2 is the person's turn, once; **c2:recall**, the reply under it carries `[reply:recall]`, names C1, and names none of A, B1 to B3. **c2:mailbox**: `support.help` shows no line carrying C1, C2 or either reply marker. Both on the open page and after the reload. -- **seg** (part 4). After leg c, `support.devices`' conversation in `support.help` holds nothing of C1 or C2. -- **e** (P2e, part 2). Post `[route:support.accounts] [scenario:needs-a-person]` and token E. **e:line**, one line answers it by `support.accounts`, carrying `[reply:escalated]`, on the open page and after the reload; **e:row-open**, within 15 s of the reading that first showed that line (of when it was due, if none did) and with no reload, the team panel's `escalations` list shows exactly one row carrying E; **e:row**, the same after the reload; **e:warning**, the boot warns that `support.help.escalations` is unattended, and warns about no other board. +- **a** (P1a). Post `[route:support.devices] [scenario:wake-after-a-hold]` and token A. Graded as **a:working**, the panel shows `support.devices is working` in a reading before the one that first shows its line; **a:line**, within 15 s of Send exactly one line answers the post, by `support.devices`, carrying `[reply:wake]`; **a:clears**, the row is gone within 5 s of that line; **a:alone**, no other seat shows as working; **a:kept**, after the reload the post and its one line are still there. +- **b** (P1b). Three posts carrying `[scenario:reply-after-a-hold]`, each after the previous answer shows and its row clears: B1 routed to `support.devices`, B2 to `support.accounts`, B3 with no route marker, for the fallback `support.general`. **b:working**, for each post the panel shows its seat working in a reading before the one that first shows its line; **b:line**, each gets exactly one line carrying its token and `[reply:in-mailbox]`, by its seat, within 15 s; **b:clears**, each post's row is gone within 5 s of its line (B3's too), and no row still shows when the next post would go; **b:alone**, no other seat shows as working, counting a row left from an earlier post; **b:only-its-own**, read from each specialist's session for the conversation, on the open page and after the reload: `devices` holds A and B1, `accounts` B2, `general` B3, each heard once with a reply under it, `fsd` holds nothing, and nobody holds another's token; **b:kept**, after the reload each post and its line show once. +- **c1** (P1c). "Talk" on `support.devices` in the roster, `[scenario:talk-to-seat]` and token C1: **c1:turn**, C1 is the person's turn, once; **c1:reply**, `[reply:talk-to-seat]` under it; on the open page and after the reload, reopened from the rail. +- **c2** (P1c). In that conversation, `[scenario:recall]` and token C2: **c2:turn**, C2 is the person's turn, once; **c2:recall**, the reply under it carries `[reply:recall]`, names C1, and names none of A, B1 to B3. **c2:coordinator**: the conversation with `support.help` shows no line carrying C1, C2 or either reply marker. Both on the open page and after the reload. +- **seg** (part 4). After leg c, `support.devices`' session for the conversation with `support.help` holds nothing of C1 or C2. -**Anti-game:** no reload before an assertion, and none but the one at each leg's end. Leg a's scenario never calls the post tool. Nothing is graded on wording past a script-written marker, a return value, a server read or a CLI run: the server is read only to know the row was written before part 2's reload. Only this run's tokens count. A seat's conversation is read only once the rail marks it current and the panel has replaced the last one. Leg c's reads are the conversation log (`[data-message-role]`), never the composer. +**Anti-game:** no reload before an assertion, and none but the one at each leg's end. Nothing is graded on wording past a script-written marker, a return value, a server read or a CLI run: the server is read only as an index (which sessions are a specialist's, which one "Talk" opened) and to wait. Only this run's tokens count. A specialist's session is read only once the rail marks it current and the panel draws that session's own messages (by the server's count and text, a wait that is not graded): the panel keeps the last session's items for a moment, and a seat panel draws a coordinator conversation's messages as turns too. Leg c's reads are the conversation log (`[data-message-role]`), never the composer. -**Model:** n/a for the gated legs: kitchen-sink's scripted model routes and answers (epic FIX-1592 D3). The run refuses to start when a provider key is set, and the server runs with `AI_GATEWAY_API_KEY` empty. The smoke alone has a key: the app's own models, its route on `vercel/typesafe-ai/jev`. +**Model:** n/a for the gated legs: kitchen-sink's scripted model scripts best fit's evaluation and answers (epic FIX-1592 D3). The run refuses to start when a provider key is set, and the server runs with `AI_GATEWAY_API_KEY` empty. The smoke alone has a key: the app's own models, its best fit on `vercel/typesafe-ai/jev`. -**Run:** `PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/run.mts`, with no provider key in the environment. It takes the plain journey, then every control below except `main`, each on a fresh `next start` and a fresh browser context, on the one build. PASS needs the plain journey green and every control behaving. +**Run:** `PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers pnpm tsx goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/run.mts`, with no provider key in the environment. It takes the plain journey, then every control below, each on a fresh `next start` and a fresh browser context, on the one build. PASS needs the plain journey green and every control behaving. -**Smoke:** `GOAL_LIVE=1` on the same command, with `AI_GATEWAY_API_KEY`: the smoke alone, out of test mode. Six posts to `support.help`, each after the previous answer and every run it started are done: a laptop that won't join the wifi, its follow-up "It sees it. It fails right after the password.", then the device, account, FSD and unclear posts. Each gets exactly one answer line within two minutes on the open page, by the specialist the fixture names, and no other seat shows as working. The follow-up's answer is by its antecedent's specialist and names the subject (one of the fixture's words); the run prints both answers. A post with no answer, or with a run still working two minutes after its answer, fails the smoke, and no later post is sent, so each answer counted is its own post's. After a reload each post sent still has its one line. Never retried: a missing answer is a finding (QR-8a). +**Smoke:** `GOAL_LIVE=1` on the same command, with `AI_GATEWAY_API_KEY`: the smoke alone, out of test mode. Six posts to `support.help`, each after the previous answer and every run it started are done: a laptop that won't join the wifi, its follow-up "It sees it. It fails right after the password.", then the device, account, FSD and unclear posts. Each gets exactly one answer line within two minutes on the open page, by a specialist, the one the fixture names where it names one, and no other seat shows as working. A post with no answer, or with a run still working two minutes after its answer, fails the smoke, and no later post is sent, so each answer counted is its own post's. After a reload each post sent still has its one line. Never retried: a missing answer is a finding (QR-8a). **Controls:** a control passes only when it reddens every assertion its row names (its own signal), fails no leg at setup, and leaves each leg it keeps green wholly green. Another red assertion in a leg its signal sits in is reported, not counted; a leg in neither list is reported and not graded. A control reddens an assertion only where the plain journey passed it on the same build: one red in both is the build's and is reported as inherited, and a named one that is inherited fails the control. | Control | Where it acts | Must FAIL, at each of these | Stays green | |---|---|---|---| -| `GOAL_CONTROL=no-live` (FIX-1609) | the page, opened with `?goalControl=no-live`: the mailbox's live view and the escalations list's | `a:line`, `b:line`, `e:row-open` (no answer, and the list shows the row only after the reload; not the working rows, which the page can still draw from the read it makes when the person posts, depending on which read wins) | c1, c2, seg | -| `GOAL_CONTROL=no-landing` (FIX-1610, kitchen-sink's from FIX-1611) | the server | `a:line`, `e:line` (a text answer never lands without a tool call) | b, c1, c2, seg | -| `GOAL_CONTROL=no-route` (FIX-1610, kitchen-sink's from FIX-1611) | the server | `a:line`, `a:alone`, `b:line`, `b:alone`, `e:line`, `e:row-open` (every member wakes: each files E and no text answer lands) | c1, c2, seg | -| `GOAL_CONTROL=drop-user-message` (FIX-1585) | this run, at the page's `fetch`: leg c's snapshot, its action streams and its live session stream lose their `user` items | `c1:turn`, `c2:turn` | a, b, seg, e | -| `GOAL_CONTROL=no-history` (FIX-1612, switch from FIX-1611) | the server | `c2:recall` | a, b, c1, seg, e | -| `GOAL_CONTROL=no-filing` (FIX-1611) | the server | `e:row-open`, `e:row` | a, b, c1, c2, seg | -| `GOAL_CONTROL=main` | today's `main`, its own build: a checkout of the last commit before FIX-1609 and FIX-1610, with this directory copied in; it reads `fixtures/today-main.json` | `a:working`, `a:line`, `b:line` | c1 | +| `GOAL_CONTROL=no-live` (FIX-1609) | the page, opened with `?goalControl=no-live`: the conversation's live view | `a:line`, `b:line` (no answer until the reload; not the working rows, which the page can still draw from the read it makes when the person posts, depending on which read wins) | c1, c2, seg | +| `GOAL_CONTROL=no-landing` (kitchen-sink's `lib/landing-control.ts`) | the server | `a:line`, `b:line` (a specialist answers in its own session and hands nothing back) | c1, c2, seg | +| `GOAL_CONTROL=no-route` (kitchen-sink's `lib/coordinator-control.ts`) | the server | `a:line`, `a:alone`, `b:line`, `b:alone` (best fit read as `everyone`: every specialist works and answers) | c1, c2, seg | +| `GOAL_CONTROL=drop-user-message` (FIX-1585) | this run, at the page's `fetch`: leg c's snapshot, its action streams and its live session stream lose their `user` items | `c1:turn`, `c2:turn` | a, b, seg | +| `GOAL_CONTROL=no-history` (FIX-1612, switch from FIX-1611) | the server | `c2:recall` | a, b, c1, seg | `drop-user-message` filters at the page's `fetch` rather than with Playwright's routing because the live session stream never ends, and a routed response is read whole before the page gets any of it. @@ -54,3 +60,6 @@ | 2026-09-28 | 60abe6e02, this directory copied in | scripted | FAIL (control, expected) | Closure run 6. `GOAL_CONTROL=main` on `fixtures/today-main.json`: failed a:working, a:line and b:line, with a:kept, b:working, b:only-its-own and b:kept also red and not counted; **c1** green. No working row is drawn at that commit, so `b:clears` passes there. | | 2026-09-28 | `main` at 15087779b | route `vercel/typesafe-ai/jev`, the app's own answer models | **PASS** (smoke) | Closure run 6, `GOAL_LIVE=1`, with the smoke now stopping, and sending no later post, when a post has no answer or a run it started is still working two minutes after it. One line each by the fixture's specialist, only that specialist seen working, and every run done before the next post: wifi, its follow-up and the phone by `support.devices` (6.0s, 5.9s, 5.9s; cleared 9.0s, 8.0s, 8.0s); the double charge by `support.accounts` (5.0s; cleared 7.0s); the FSD question by `support.fsd` (8.8s; cleared 8.8s); the parking pass by `support.general` (4.9s; cleared 7.0s). The follow-up's answer names the password. All six kept after the reload. | | 2026-09-28 | `main` at 15087779b, its own checkout | a key; routing on the app's own route, the specialists on `vercel/openai/gpt-5-nano` (the container sets `FSDEV_INTENT_*`, which routing does not read) | **PASS** (docs smoke, part 4, ER-19) | Blind: an agent that had read no spec, goal, internal doc or history followed only `apps/kitchen-sink/README.md` and the mailboxes guide's routing and on-screen sections, as written, on port 3000. All six tasks passed: the app ran as the README says; the wifi post showed `support.devices is working` at +2.2s and its answer at +12.3s, with no reload; each post, sent after the previous answer, got one answer, the follow-up from `support.devices` like its antecedent, the double charge from `support.accounts`, the FSD question from `support.fsd`, the parking pass from `support.general` (the evaluator's pick); a direct conversation with `support.devices` recalled its code word, kept it after a reload and showed none of it in the mailbox; a needs-a-person post filed a row onto `escalations`, live in two tabs (+14.4s, +14.8s), `pending`, newest first, kept after a reload; and what it saw matched the guide (another tab's post in about a second, routing held or evaluated as described, no route item drawn as a line). Nine doc gaps, none blocking a flow the legs or the smoke use: FIX-1631 (QR-14a, non-blocking), fixed on #2350. Outside the epic: FIX-1632 (the boot warning repeats on each `next dev` hot reload, QR-16) and FIX-1633 (the page header names the declared model while an intent override runs another). It replaces the closure run's own docs check, which read the spec first. | +| 2026-10-09 | `main` at 57d2c10e9 | scripted | **FAIL** | **Rewritten for FIX-1792 P1; the check before the rewrite taken first, on clean `main`.** Plain journey: a and e green; **b** and **c1** could not be walked (`b:setup`, `c1:setup`: each waited for a rail row named for a specialist, and since FIX-1788 P4 the rail lists one `agent` copy and no row per worker). Under every control the same two legs could not be walked, so `drop-user-message` and `no-history` left their signals green. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | FAIL (check defect) | **Rewritten**, first run. **b:only-its-own** read the conversation with `support.help` as `support.devices`' session: the panel still drew the coordinator conversation's messages when the rail marked the session current. The check now waits until the panel draws that session's own messages (by the server's read of it, not graded). `c1:setup` also red: "Talk" was pressed with a seat panel already open, so the composer was found before the conversation was. Leg c now starts from the coordinator's panel. And the worker flow's kind and its copy, both named `agent`, are now found by their own attributes, not by name. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Rewritten.** One fresh build. **Plain journey**, every assertion on the open page and after its reload: a, `support.devices` working +0.9s, its line +4.0s, cleared +4.0s, nobody else working; b, each seat working +0.9s before its one line at +3.9s to +4.0s, by `support.devices`, `support.accounts`, `support.general`, each specialist's session for the conversation holding only its own posts and `support.fsd` none; c1 and c2 at +0.1s, the recall naming C1 only, the conversation with `support.help` showing none of it; seg. **Controls**, each failing every assertion it names, no leg at setup, and its green legs green: `no-live` a:line, b:line (a:working, b:working also red, not counted); `no-landing` a:line, b:line (a:kept, b:working, b:alone, b:only-its-own, b:kept also red, not counted: with no answer landing, best fit holds every later post for `support.devices`); `no-route` a:line, a:alone, b:line, b:alone (a:kept, b:only-its-own, b:kept also red, not counted); `drop-user-message` c1:turn, c2:turn; `no-history` c2:recall. | diff --git a/specs/issues/FIX-1792/poc/inventory/check.mjs b/specs/issues/FIX-1792/poc/inventory/check.mjs index 767f817ff8..53cec57c44 100644 --- a/specs/issues/FIX-1792/poc/inventory/check.mjs +++ b/specs/issues/FIX-1792/poc/inventory/check.mjs @@ -297,7 +297,7 @@ const FILE_CLASS = { "apps/docs/docs/workforce/workforce-overview.svg": "E · S12 · the figure draws a MAILBOX.md", "apps/kitchen-sink/README.md": "C · S6 · the help desk is a coordinator; the escalation feature goes", "apps/kitchen-sink/app/page.tsx": "C · S6 · the page talks to the help coordinator", - "apps/kitchen-sink/components/flow-state/chat-assistant.tsx": "E · S6 · renders coordinator-route, not mailbox-route", + "apps/kitchen-sink/components/flow-state/chat-assistant.tsx": "E · S9 · with the registry copy it is installed from (upstream first): mailbox-route goes; no coordinator-route item reaches this renderer, the coordinator panel draws messages only", "apps/kitchen-sink/components/picked-session-panel.tsx": "C · S6 · the picked conversation is the coordinator's", "apps/kitchen-sink/e2e/talk-from-page.spec.ts": "C · S6 · posts through the coordinator; the escalation leg goes", "apps/kitchen-sink/e2e/workforce-shell.spec.ts": "C · S6 · the escalations panel goes", From b767c8c4e5b8d03d91a2acd5e5108a22759ecdb1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 01:30:30 +0000 Subject: [PATCH 04/14] docs(goals): the live legs' verdicts on the coordinator (FIX-1792 P1) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- .../a-person-talks-to-a-seat-a-mailbox-and-back/goal.md | 1 + goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md | 1 + 2 files changed, 2 insertions(+) diff --git a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md index 2fbd018998..f54e47e9e1 100644 --- a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md +++ b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md @@ -63,3 +63,4 @@ | 2026-10-09 | `main` at 57d2c10e9 | scripted | **FAIL** | **Rewritten for FIX-1792 P1; the check before the rewrite taken first, on clean `main`.** Plain journey: a and e green; **b** and **c1** could not be walked (`b:setup`, `c1:setup`: each waited for a rail row named for a specialist, and since FIX-1788 P4 the rail lists one `agent` copy and no row per worker). Under every control the same two legs could not be walked, so `drop-user-message` and `no-history` left their signals green. | | 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | FAIL (check defect) | **Rewritten**, first run. **b:only-its-own** read the conversation with `support.help` as `support.devices`' session: the panel still drew the coordinator conversation's messages when the rail marked the session current. The check now waits until the panel draws that session's own messages (by the server's read of it, not graded). `c1:setup` also red: "Talk" was pressed with a seat panel already open, so the composer was found before the conversation was. Leg c now starts from the coordinator's panel. And the worker flow's kind and its copy, both named `agent`, are now found by their own attributes, not by name. | | 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Rewritten.** One fresh build. **Plain journey**, every assertion on the open page and after its reload: a, `support.devices` working +0.9s, its line +4.0s, cleared +4.0s, nobody else working; b, each seat working +0.9s before its one line at +3.9s to +4.0s, by `support.devices`, `support.accounts`, `support.general`, each specialist's session for the conversation holding only its own posts and `support.fsd` none; c1 and c2 at +0.1s, the recall naming C1 only, the conversation with `support.help` showing none of it; seg. **Controls**, each failing every assertion it names, no leg at setup, and its green legs green: `no-live` a:line, b:line (a:working, b:working also red, not counted); `no-landing` a:line, b:line (a:kept, b:working, b:alone, b:only-its-own, b:kept also red, not counted: with no answer landing, best fit holds every later post for `support.devices`); `no-route` a:line, a:alone, b:line, b:alone (a:kept, b:only-its-own, b:kept also red, not counted); `drop-user-message` c1:turn, c2:turn; `no-history` c2:recall. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15 | route `vercel/typesafe-ai/jev`, the app's own answer models | **PASS** (smoke) | `GOAL_LIVE=1`. One line each within two minutes, only the answering specialist seen working, every run done before the next post: the wifi post by `support.devices` (2.9s); its follow-up by `support.accounts` (6.0s), which answered *I don't have enough context to diagnose this — I don't know … what "it" is* (the open question on the P1 PR); the phone by `support.devices` (2.9s), which called it *a separate issue from the wifi problem*; the double charge by `support.accounts` (3.9s); the FSD question by `support.fsd` (2.9s); the parking pass by `support.general` (2.9s). All six kept after the reload. | diff --git a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md index 60a2252666..3f67fad997 100644 --- a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md +++ b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md @@ -55,3 +55,4 @@ Since FIX-1788 the rail lists each flow, not each worker, so "four specialists n | 2026-10-09 | `main` at 57d2c10e9 | scripted | FAIL | **Rewritten for FIX-1792 P1; the check before the rewrite taken first, on clean `main`.** It failed at **roster**: the rail's Seats section lists the kinds `[]` (since FIX-1788 P4 the section is Workers, and holds one `agent` copy). | | 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Rewritten.** Fresh production build, keyless. **roster:** the rail lists `coordinator`, opening on `["support.help"]`, and `agent`; the roster panel lists `support.help` (coordinator) *Ask the support team anything.* and the four specialists (agent), each with its description as the fixture has it; no "Hire another". **answer:** after a reload, one line answering the post, by `support.devices`: *[reply:wake] Heard it.*, without the token. | | 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-landing`: **answer** only — after the reload no line answers the token. **roster** green. | +| 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15 | scripted, then the app's own models | **PASS** | `GOAL_LIVE=1`, same build served out of test mode. Scripted legs as above. Live: the swollen battery answered by `support.devices`; the triple charge and the taken-over account by `support.accounts`; the wifi password by `support.devices`; the invoices by `support.accounts`. One answer line each. The taken-over account's answer says *I'm escalating this to our security/support team*; nothing escalates now. | From 92d8fab3942b3afced5aaa2f92d23c889656cbe9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 01:39:15 +0000 Subject: [PATCH 05/14] feat(goals): the asker lab's front is a coordinator (FIX-1792 P1) Built on fix/closure-children-one-copy's hire fix for the asker lab (#2933). `front` is a WORKER.md on the coordinator flow (delegates: [desk.asker], routing: everyone); the asker takes no delegated posts, so a post to it reaches nobody, as the mailbox woke nobody. The lab registers the coordinator flow, opens no mailbox, and writes the inventory from a flow of its own. The check finds the asker by its flow. The inventory checker marks the file done and classifies the concept-doc skill's example. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- .../goal.md | 1 + .../lab/asker/fsdev.config.mts | 80 ++++++++----------- .../MAILBOX.md => workers/front/WORKER.md} | 4 +- .../run.mts | 5 +- specs/issues/FIX-1792/poc/inventory/check.mjs | 3 +- 5 files changed, 42 insertions(+), 51 deletions(-) rename goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/workforce/teams/desk/{mailboxes/front/MAILBOX.md => workers/front/WORKER.md} (54%) diff --git a/goals/shift-manager/it-sends-a-turn-into-a-seat-session/goal.md b/goals/shift-manager/it-sends-a-turn-into-a-seat-session/goal.md index 9aa5b33363..0471624831 100644 --- a/goals/shift-manager/it-sends-a-turn-into-a-seat-session/goal.md +++ b/goals/shift-manager/it-sends-a-turn-into-a-seat-session/goal.md @@ -56,3 +56,4 @@ Routes: **task composer** (the first row, reached from Tasks by clicking), **@co | 2026-10-02 | feat/FIX-1737-d-inbox-tasks-roster (pre-PR) | n/a | PASS | **heard** now also requires Inbox to draw the reply under the ask, read back from the ask's session: drawn. Both coding routes delivered with the line held, request `aborted`, same coding session resumed, standing 1 → 1; EM reply disabled. | | 2026-10-02 | feat/FIX-1737-d-inbox-tasks-roster (pre-PR), `GOAL_CONTROL=optimistic-turn` | n/a | FAIL (expected) | **delivered** on the sending routes, and on the Inbox reply **heard** now also names the reply never drawn under the ask. | | 2026-10-09 | fix/closure-children-one-copy (main 57d2c10e9) | n/a | PASS | The asker Lab builds an installation and registers one copy of `asker`, and the ask is raised in a session on `asker` naming `desk.asker` (FIX-1788 P4). Before, the asker config failed to load: "installation.workerFlows is not a function". `optimistic-turn` FAILs at **delivered**; `fresh-session` only at **continued**. | +| 2026-10-09 | `fix/FIX-1792-p1` on fix/closure-children-one-copy 55e02d269, uncommitted | n/a | PASS | FIX-1792 P1: the asker Lab's `front` is a coordinator `WORKER.md` (`delegates: [desk.asker]`, `routing: everyone`), its config registers Workforce's `coordinator` flow and writes the inventory from a flow of its own, and the check finds the asker by its flow rather than as the tree's first worker. Every route delivered; the Inbox reply on `desk.asker` was heard in the ask's session. `optimistic-turn` FAILs at **delivered** (and what follows from no delivery: stopped, continued, standing, em door, heard); `fresh-session` only at **continued**. | diff --git a/goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/fsdev.config.mts b/goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/fsdev.config.mts index 7cf8d40089..5efdd00b12 100644 --- a/goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/fsdev.config.mts +++ b/goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/fsdev.config.mts @@ -1,22 +1,24 @@ /** * The goal's fixture Lab, as Shift Manager's command loads it: one seat that - * asks and hears (`asker.mts`), in one mailbox, with the organization's - * inventory open so the seat's row names its door. In-memory stores and no - * credential, so every start is fresh. The check raises the ask itself, - * through the Lab's own action route. + * asks and hears (`asker.mts`), beside one coordinator, with the + * organization's inventory open so the seat's row names its door. In-memory + * stores and no credential, so every start is fresh. The check raises the ask + * itself, through the Lab's own action route. + * + * The coordinator, `desk.front`, is a worker file on Workforce's `coordinator` + * flow. The asker's flow takes no delegated posts, so a post to `desk.front` + * reaches nobody, as the mailbox it replaced woke nobody. */ import { createFlowState, inMemoryStores, runAction } from "@flow-state-dev/engine"; +import { defineFlow } from "@flow-state-dev/core"; import type { FlowInstance } from "@flow-state-dev/core/types"; import { - MAILBOX_KIND, - mailboxBoardIds, - mailboxInstances, - defineMailboxFlow, + COORDINATOR_KIND, + defineCoordinatorFlow, inventorySeats, - openMailboxes, + inventoryWriterActions, openInventory, type InventoryActionRequest, - type OpenMailboxesOptions, } from "@flow-state-dev/workforce"; import { readDeclaredRoster } from "@flow-state-dev/workforce/loader"; import { DEFAULT_ORG_ID } from "@flow-state-dev/core"; @@ -27,16 +29,30 @@ import { ASKER_KIND, defineAskerFlow } from "./asker.mts"; /** The one person this Lab runs as. */ const USER_ID = "u_turn_goal"; +/** + * The model best fit's one evaluator call would run on. `desk.front` routes + * to everyone and never makes that call, but the flow names no default. + */ +const ROUTE_MODEL = "vercel/openai/gpt-5.4-mini"; + +/** The flow the inventory's seat rows are written from: the Lab's own, carrying the writer. */ +const INVENTORY_WRITER_KIND = "inventory-writer"; + const tree = await readDeclaredRoster(fileURLToPath(new URL("./workforce", import.meta.url))); if (tree.problems.length > 0) throw new Error(`the asker tree did not load: ${tree.problems.map((p) => p.error.message).join("; ")}`); -const { installation, copies } = installWorkers(tree.workers, (installation) => ({ [ASKER_KIND]: defineAskerFlow(installation) }), { - mailboxBoards: mailboxBoardIds(tree.mailboxes), -}); -const mailboxKind = defineMailboxFlow({ inventory: true }); +const { installation, copies } = installWorkers(tree.workers, (installation) => ({ + [ASKER_KIND]: defineAskerFlow(installation), + [COORDINATOR_KIND]: defineCoordinatorFlow({ installation, delegateFlows: [], routeModel: ROUTE_MODEL }), +})); +// The seat write takes caller-supplied rows, so it is internal only. +const writer = defineFlow({ + kind: INVENTORY_WRITER_KIND, + internal: { actions: { ...inventoryWriterActions(INVENTORY_WRITER_KIND) } }, +} as never) as unknown as FlowInstance; const flows: Record = { - ...Object.fromEntries(mailboxInstances(tree.mailboxes, { kinds: { [MAILBOX_KIND]: mailboxKind as never } }).map((i) => [i.kind, i])), ...Object.fromEntries(copies.map((copy) => [copy.id, copy])), + [INVENTORY_WRITER_KIND]: writer, }; const flowState = createFlowState({ flows, @@ -45,39 +61,9 @@ const flowState = createFlowState({ devtool: { userId: USER_ID }, } as never); -const router = (await flowState.getRouter()) as Record Promise>; -const call = async (method: "GET" | "POST" | "DELETE", path: string[], body?: unknown) => { - const response = await router[method]!( - new Request(`http://turn-goal.local/api/flows/${path.map(encodeURIComponent).join("/")}`, { - method, - headers: { "content-type": "application/json" }, - ...(body === undefined ? {} : { body: JSON.stringify(body) }), - }), - { params: { path } }, - ); - const text = await response.text(); - return { status: response.status, body: text.length > 0 ? JSON.parse(text) : null }; -}; -const client: OpenMailboxesOptions["client"] = { - createSession: async (create) => { - const { status, body } = await call("POST", [create.flowKind, "sessions"], create); - if (status >= 400) throw Object.assign(new Error(`create session: ${status}`), { status }); - return body; - }, - getSession: async (sessionId) => { - const { status, body } = await call("GET", ["sessions", sessionId]); - if (status >= 400) throw new Error(`read session ${sessionId}: ${status}`); - return body?.session ?? body; - }, - deleteSession: async (sessionId) => { - await call("DELETE", ["sessions", sessionId]); - }, -}; -await openMailboxes(tree.mailboxes, { client, userId: USER_ID }); - const runtime = await flowState.getRuntime(); const opened = await openInventory( - { seats: inventorySeats(installation), mailboxes: tree.mailboxes }, + { seats: inventorySeats(installation), mailboxes: [] }, { run: async (request: InventoryActionRequest) => { const result = (await runAction({ @@ -94,7 +80,7 @@ const opened = await openInventory( if (result?.error !== undefined) throw new Error(String((result.error as Error).message ?? result.error)); return result; }, - seatWriter: { flowKind: MAILBOX_KIND }, + seatWriter: { flowKind: INVENTORY_WRITER_KIND }, userId: USER_ID, orgId: DEFAULT_ORG_ID, }, diff --git a/goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/workforce/teams/desk/mailboxes/front/MAILBOX.md b/goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/workforce/teams/desk/workers/front/WORKER.md similarity index 54% rename from goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/workforce/teams/desk/mailboxes/front/MAILBOX.md rename to goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/workforce/teams/desk/workers/front/WORKER.md index 3767f0f16b..42903de9af 100644 --- a/goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/workforce/teams/desk/mailboxes/front/MAILBOX.md +++ b/goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/workforce/teams/desk/workers/front/WORKER.md @@ -1,6 +1,8 @@ --- description: Where the desk takes requests. -members: [desk.asker] +flow: coordinator +delegates: [desk.asker] +routing: everyone --- Requests land here. diff --git a/goals/shift-manager/it-sends-a-turn-into-a-seat-session/run.mts b/goals/shift-manager/it-sends-a-turn-into-a-seat-session/run.mts index 2632cc1cb5..e60e6202e2 100644 --- a/goals/shift-manager/it-sends-a-turn-into-a-seat-session/run.mts +++ b/goals/shift-manager/it-sends-a-turn-into-a-seat-session/run.mts @@ -58,7 +58,7 @@ import { seatSessionId } from "../../../packages/shift-manager/teams/devteam/ask import { REPO_ROOT, goalTmpDir, intentFreeEnv, runGoal } from "../../lib/index.mts"; import { SHIFT_MANAGER_COMMAND, servedAddresses } from "../../lib/shift-manager.mts"; import { launchChromium } from "../../lib/playwright.mts"; -import { heardLine } from "./lab/asker/asker.mts"; +import { ASKER_KIND, heardLine } from "./lab/asker/asker.mts"; import type { RecordedAttempt } from "./lab/recording-harness.mts"; import { ROWS, STORE_ENV } from "./lab/rows.mts"; @@ -384,7 +384,8 @@ await runGoal(async () => { const where: Where = { mailboxId: mailbox.id, boardRef: `${mailbox.id}.${boardName}`, ledgerId: mailboxBoard(mailbox.id, boardName).id }; const coder = devteam.workers.find((w) => w.declared.flow === "coder" && (mailbox.declared.members as string[]).includes(w.id) && w.id.endsWith(".coder"))!; const em = devteam.workers.find((w) => w.declared.flow === "em")!; - const asker = (await readDeclaredRoster(LABS.asker.tree)).workers[0]!; + // The asker lab's one worker on the asker kind; its coordinator, `desk.front`, is another worker file. + const asker = (await readDeclaredRoster(LABS.asker.tree)).workers.find((w) => w.declared.flow === ASKER_KIND)!; const pages = process.env.GOAL_PAGES ?? (await buildShiftManager(CONTROL)); const failures: string[] = []; diff --git a/specs/issues/FIX-1792/poc/inventory/check.mjs b/specs/issues/FIX-1792/poc/inventory/check.mjs index 53cec57c44..469159dba0 100644 --- a/specs/issues/FIX-1792/poc/inventory/check.mjs +++ b/specs/issues/FIX-1792/poc/inventory/check.mjs @@ -199,7 +199,7 @@ const FILES = { "goals/shift-manager/it-draws-v2s-look/lab/workforce/teams/desk/mailboxes/front/MAILBOX.md": { target: "session", boards: ["work"], use: "every screen draws the board's rows in v2's look" }, "goals/shift-manager/it-sends-a-turn-into-a-seat-session/lab/asker/workforce/teams/desk/mailboxes/front/MAILBOX.md": - { target: "coordinator", boards: [], use: "a person's line lands in a worker's session" }, + { done: "P1", target: "coordinator", boards: [], use: "a person's line lands in a worker's session" }, "goals/shift-manager/it-shows-who-is-on-shift/lab/workforce/teams/eng/mailboxes/desk/MAILBOX.md": { target: "session", boards: ["work"], use: "the roster shows the tasks each worker holds; the lead drains it" }, "goals/shift-manager/it-shows-who-is-on-shift/lab/workforce/teams/ops/mailboxes/desk/MAILBOX.md": @@ -272,6 +272,7 @@ const FILE_CLASS = { "packages/workforce/src/best-fit.ts": "E · S9 · a comment names the mailbox's route wrapper, routeByPurpose", "packages/shift-manager/test/devteam-cos-tool-names.test.ts": "E · S7 · the chief of staff loses setWorkstreams", "packages/shift-manager/test/devteam-chief-of-staff-coordinator.test.ts": "C · S7 · the DevTeam host's mailboxes option in a test; the host moves off mailboxes", + ".agents/skills/concept-doc/example/CONCEPT.md": "E · S12 · the concept-doc skill's worked example names removed exports; P4 edits it or `--after` fails on it", ".omp/extensions/mailbox.ts": "U · — · the agent-mailbox tool's command, not Workforce", "apps/docs/docs/client/react.md": "E · S12 · an example opens a session on flowKind \"mailbox\"", "apps/docs/docs/devtool/overview.md": "E · S12 · the inventory view's registered-mailboxes row", From 482afa10338c03d200d0ed38c7d6f95ee0545c84 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 01:39:15 +0000 Subject: [PATCH 06/14] refactor(kitchen-sink): coordinatorLines is the panel's own (FIX-1792 P1) Nothing imports it or its line type, so neither is exported. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- apps/kitchen-sink/components/picked-session-panel.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/kitchen-sink/components/picked-session-panel.tsx b/apps/kitchen-sink/components/picked-session-panel.tsx index b1777a9949..2c87117be2 100644 --- a/apps/kitchen-sink/components/picked-session-panel.tsx +++ b/apps/kitchen-sink/components/picked-session-panel.tsx @@ -37,7 +37,7 @@ type Session = ReturnType; export type RequestStatusLookup = (requestId: string) => Promise; /** One line of a coordinator's conversation: who wrote it, and what it says. */ -export interface CoordinatorLine { +interface CoordinatorLine { id: string; /** A delegate's worker id, the person's id, or, for the coordinator's own word, the coordinator. */ label: string; @@ -53,7 +53,7 @@ export interface CoordinatorLine { * @param items The session's items. * @param person The name the person's own posts are drawn under. */ -export function coordinatorLines(items: Session["items"], person: string): CoordinatorLine[] { +function coordinatorLines(items: Session["items"], person: string): CoordinatorLine[] { return items.flatMap((item) => { const message = item as { type: string; From 6c912d3fec6a001b7829ba1b63fee5e852e26e86 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 01:47:13 +0000 Subject: [PATCH 07/14] fix(kitchen-sink): the support specialists never promise a hand-off (FIX-1792 P1) With the escalation feature removed, a specialist on a real model still told a locked-out user it was escalating the case, and nothing does that. Each specialist's instructions gain one sentence: never promise to escalate or hand a case to another team; answer what you can, and say plainly what you can't. No tool is added (BR-14). answers-a-clerk-note's live leg re-run: none of its five answers promises a hand-off. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- .../workforce/teams/support/workers/accounts/WORKER.md | 1 + .../workforce/teams/support/workers/devices/WORKER.md | 1 + apps/kitchen-sink/workforce/teams/support/workers/fsd/WORKER.md | 1 + .../workforce/teams/support/workers/general/WORKER.md | 1 + goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md | 1 + 5 files changed, 5 insertions(+) diff --git a/apps/kitchen-sink/workforce/teams/support/workers/accounts/WORKER.md b/apps/kitchen-sink/workforce/teams/support/workers/accounts/WORKER.md index 62d7bf037d..eebc555f22 100644 --- a/apps/kitchen-sink/workforce/teams/support/workers/accounts/WORKER.md +++ b/apps/kitchen-sink/workforce/teams/support/workers/accounts/WORKER.md @@ -4,3 +4,4 @@ description: Sign-in, passwords, billing, refunds and subscriptions. You are the support team's accounts specialist. Answer in a sentence or two, and say plainly when you don't know. +Never promise to escalate a case or hand it to another team: answer what you can, and say plainly what you can't do. diff --git a/apps/kitchen-sink/workforce/teams/support/workers/devices/WORKER.md b/apps/kitchen-sink/workforce/teams/support/workers/devices/WORKER.md index 3f7f3b5353..ba560827b9 100644 --- a/apps/kitchen-sink/workforce/teams/support/workers/devices/WORKER.md +++ b/apps/kitchen-sink/workforce/teams/support/workers/devices/WORKER.md @@ -4,3 +4,4 @@ description: Printers, laptops, phones, wifi and anything else with a power butt You are the support team's devices specialist. Answer in a sentence or two, and say plainly when you don't know. +Never promise to escalate a case or hand it to another team: answer what you can, and say plainly what you can't do. diff --git a/apps/kitchen-sink/workforce/teams/support/workers/fsd/WORKER.md b/apps/kitchen-sink/workforce/teams/support/workers/fsd/WORKER.md index d39949526e..31d25cec94 100644 --- a/apps/kitchen-sink/workforce/teams/support/workers/fsd/WORKER.md +++ b/apps/kitchen-sink/workforce/teams/support/workers/fsd/WORKER.md @@ -4,3 +4,4 @@ description: Questions about building apps with the flow-state-dev framework. You are the support team's flow-state-dev specialist. Answer in a sentence or two, and say plainly when you don't know. +Never promise to escalate a case or hand it to another team: answer what you can, and say plainly what you can't do. diff --git a/apps/kitchen-sink/workforce/teams/support/workers/general/WORKER.md b/apps/kitchen-sink/workforce/teams/support/workers/general/WORKER.md index 0d46fa190a..cf5600319f 100644 --- a/apps/kitchen-sink/workforce/teams/support/workers/general/WORKER.md +++ b/apps/kitchen-sink/workforce/teams/support/workers/general/WORKER.md @@ -4,3 +4,4 @@ description: Anything that fits none of the other specialists. You are the support team's general specialist: you take what the others don't. Answer in a sentence or two, and say plainly when you don't know. +Never promise to escalate a case or hand it to another team: answer what you can, and say plainly what you can't do. diff --git a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md index 3f67fad997..e499f30f38 100644 --- a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md +++ b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md @@ -56,3 +56,4 @@ Since FIX-1788 the rail lists each flow, not each worker, so "four specialists n | 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Rewritten.** Fresh production build, keyless. **roster:** the rail lists `coordinator`, opening on `["support.help"]`, and `agent`; the roster panel lists `support.help` (coordinator) *Ask the support team anything.* and the four specialists (agent), each with its description as the fixture has it; no "Hire another". **answer:** after a reload, one line answering the post, by `support.devices`: *[reply:wake] Heard it.*, without the token. | | 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-landing`: **answer** only — after the reload no line answers the token. **roster** green. | | 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15 | scripted, then the app's own models | **PASS** | `GOAL_LIVE=1`, same build served out of test mode. Scripted legs as above. Live: the swollen battery answered by `support.devices`; the triple charge and the taken-over account by `support.accounts`; the wifi password by `support.devices`; the invoices by `support.accounts`. One answer line each. The taken-over account's answer says *I'm escalating this to our security/support team*; nothing escalates now. | +| 2026-10-09 | `fix/FIX-1792-p1` at 482afa103, uncommitted: one sentence added to each support specialist's instructions, *Never promise to escalate a case or hand it to another team: answer what you can, and say plainly what you can't do.* | scripted, then the app's own models | **PASS** | `GOAL_LIVE=1`, re-taken for the row above, whose taken-over-account answer promised an escalation nothing makes. Scripted legs as above. Live: the swollen battery by `support.devices` (*I'm not able to arrange or guarantee a same-day replacement myself*); the triple charge by `support.accounts` (*I can't arrange for someone else to review this case*); the taken-over account by `support.accounts` (*I can't arrange for another person or team to do it either*, then the service's own recovery process); the wifi password by `support.devices`; the invoices by `support.accounts`. One answer line each, and none promises an escalation or a hand-off. | From 343af0c776913d1eab9f645c6e569cc024774a20 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 16:43:37 +0000 Subject: [PATCH 08/14] chore(specs): the FIX-1792 checker classifies FIX-1791's goal check (P3) goals/coordinators/hands-each-post-to-its-delegates arrived from main and reads the DevTeam's mailbox boards; it follows the DevTeam in P3 (S7). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- specs/issues/FIX-1792/poc/inventory/check.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/issues/FIX-1792/poc/inventory/check.mjs b/specs/issues/FIX-1792/poc/inventory/check.mjs index 469159dba0..774f349f5f 100644 --- a/specs/issues/FIX-1792/poc/inventory/check.mjs +++ b/specs/issues/FIX-1792/poc/inventory/check.mjs @@ -484,6 +484,7 @@ const FILE_CLASS = { const GOALS = { "goals/README.md": "EDIT · the word, FIX-1796", "goals/lib/workers.mts": "EDIT · the hire's mailboxBoards option goes with S9", + "goals/coordinators/hands-each-post-to-its-delegates": "CONVERT · FIX-1791's goal check reads the DevTeam's mailbox boards; it follows the DevTeam in P3 (S7)", "goals/agent-discovery/an-orchestrator-routes-a-task-by-asking": "EDIT · a roster literal's mailboxes field", "goals/design-system/skins-reused-components-from-one-token-set": "EDIT · a rail label in a demo host", "goals/devforce-lab/it-codes-in-the-projects-repository": "CONVERT · the run finds its project through the feature workstream", From 9e5cfb5177e33cbff492d6c5583894a2a8a8b346 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 16:55:25 +0000 Subject: [PATCH 09/14] chore(specs): the FIX-1792 checker's file census names only files that exist (P1) FILE_CLASS still classified twelve files that are gone: the five kitchen-sink controls and the escalation and notify files P1 removed (their replacements, coordinator-control.ts and landing-control.ts, name none of the removed surface, so they need no row), and four FIX-1788 removed before it. The census printed them as "classified but gone"; it now describes the tree. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- specs/issues/FIX-1792/poc/inventory/check.mjs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/specs/issues/FIX-1792/poc/inventory/check.mjs b/specs/issues/FIX-1792/poc/inventory/check.mjs index 774f349f5f..bae7aa0ba9 100644 --- a/specs/issues/FIX-1792/poc/inventory/check.mjs +++ b/specs/issues/FIX-1792/poc/inventory/check.mjs @@ -304,16 +304,10 @@ const FILE_CLASS = { "apps/kitchen-sink/e2e/workforce-shell.spec.ts": "C · S6 · the escalations panel goes", "apps/kitchen-sink/fsdev.config.ts": "C · S6 · mailbox wiring goes; the coordinator flow is on the list", "apps/kitchen-sink/lib/e2e-mock-script.ts": "C · S6 · the scripted replies name the coordinator; the escalation replies go", - "apps/kitchen-sink/lib/escalate-control.ts": "R · S6 · the no-filing control goes with the escalation feature", - "apps/kitchen-sink/lib/mailbox-landing-control.ts": "C · S6 · a goal control re-expressed on the coordinator, or gone with its leg", - "apps/kitchen-sink/lib/mailbox-post-control.ts": "C · S6 · a goal control re-expressed on the coordinator, or gone with its leg", - "apps/kitchen-sink/lib/mailbox-route-control.ts": "C · S6 · a goal control re-expressed on the coordinator, or gone with its leg", - "apps/kitchen-sink/lib/mailbox-wake-control.ts": "C · S6 · a goal control re-expressed on the coordinator, or gone with its leg", "apps/kitchen-sink/lib/models.ts": "E · S6 · a comment names mailbox-route", "apps/kitchen-sink/lib/workforce-shell.ts": "C · S6 · the escalations board ref and panel go", "apps/kitchen-sink/test/client-imports.test.ts": "E · S6 · MAILBOX_POST_COMPONENT import", "apps/kitchen-sink/test/e2e-mock-script.test.ts": "C · S6 · tests", - "apps/kitchen-sink/test/escalate.test.ts": "R · S6 · tests of the removed escalate tool", "apps/kitchen-sink/test/goal-control.test.ts": "C · S6 · tests; the no-filing control goes", "apps/kitchen-sink/test/mailbox-reply.test.ts": "C · S6 · a delegate's answer, not post-to-mailbox", "apps/kitchen-sink/test/mailbox-talk.test.ts": "C · S6 · tests", @@ -323,9 +317,7 @@ const FILE_CLASS = { "apps/kitchen-sink/test/picked-session-panel.test.tsx": "C · S6 · tests", "apps/kitchen-sink/test/support-desk.test.ts": "C · S6 · the unattended-board warning goes", "apps/kitchen-sink/test/workforce-shell.test.ts": "C · S6 · tests", - "apps/kitchen-sink/workforce/blocks/escalate.ts": "R · S6 · the escalation feature is removed, not replaced", "apps/kitchen-sink/workforce/hire.ts": "C · S6 · mailbox binder calls and the escalate control go", - "apps/kitchen-sink/workforce/mailbox-notify.ts": "R · S6 · the mailbox's notify block", "apps/kitchen-sink/workforce/teams/support/workers/accounts/WORKER.md": "C · S6 · post-to-mailbox and escalate leave its tools, with the escalation instruction", "apps/kitchen-sink/workforce/teams/support/workers/devices/WORKER.md": "C · S6 · post-to-mailbox and escalate leave its tools, with the escalation instruction", "apps/kitchen-sink/workforce/teams/support/workers/fsd/WORKER.md": "C · S6 · post-to-mailbox and escalate leave its tools, with the escalation instruction", @@ -366,7 +358,6 @@ const FILE_CLASS = { "packages/shift-manager/test/build-inputs.test.ts": "E · S8 · tests", "packages/shift-manager/test/chief-of-staff.test.ts": "E · S8 · workstreams as mailboxes in a fixture", "packages/shift-manager/test/columns.test.ts": "E · S8 · tests", - "packages/shift-manager/test/devforce-lab-hired-boot.test.ts": "C · S7 · tests", "packages/shift-manager/test/devteam-legacy-org.test.ts": "E · S7 · the mailbox session leaves its old-org fixture", "packages/shift-manager/test/devteam-pre-rename-store.test.ts": "R · S3 · the pre-rename store check goes with the refusal (D2)", "packages/shift-manager/test/devteam-repository-ask.test.ts": "C · S7 · the project's workstream, not a claim", @@ -426,7 +417,6 @@ const FILE_CLASS = { "packages/workforce/src/projects/project-writes.ts": "E · S10 · setWorkstreams and claims go", "packages/workforce/src/projects/talk-template.ts": "T · — · FIX-1793 removes it with rooms", "packages/workforce/src/projects/talk.ts": "T · — · FIX-1793 removes it with rooms", - "packages/workforce/src/seat-hire-blocks.ts": "E · S9 · the mailboxBoards hire option", "packages/workforce/test/agent-routed-answer.test.ts": "R · S9 · a routed mailbox answer; FIX-1791's delegated post replaces it", "packages/workforce/test/agent-worker-history.test.ts": "E · S9 · its wake fixture", "packages/workforce/test/agent-worker-mailbox-post.test.ts": "R · S9 · tests of the agent's onMailboxPost entry; FIX-1791 S9's delegated post replaces it", @@ -465,8 +455,6 @@ const FILE_CLASS = { "packages/workforce/test/resources-doors-characterization.test.ts": "E · S1 · mailboxes/ decoy paths", "packages/workforce/test/seat-discovery.test.ts": "E · S9 · the mailboxes domain goes", "packages/workforce/test/seat-door.test.ts": "E · S9 · the roster's mailboxes field", - "packages/workforce/test/seat-hire-capability.test.ts": "E · S9 · the mailboxBoards option", - "packages/workforce/test/seat-id.test.ts": "E · S9 · tests", "packages/workforce/test/symlink-containment-matrix.test.ts": "E · S1 · tests", "packages/workforce/test/wake-member-seats.test-d.ts": "R · S9 · tests of the wake", "packages/workforce/test/wake-member-seats.test.ts": "R · S9 · tests of the wake", From 4b597367f0eeaffd4075bf3a95c27ba05057cbc4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 16:55:25 +0000 Subject: [PATCH 10/14] fix(goals): the fresh-host and routed-post hosts run as a named org (FIX-1792 P1, BR-25) Neither converted host had a resolvePrincipal, so every session they made, the coordinator conversation and each delegate's, landed in the development default org, and neither goal could see a session leave the named-org path. Each host now names its one user and org in resolvePrincipal, ignoring the request (BP-031). Each check gains an org leg that reads the conversation and every delegate session under it from the store (the router scopes listings to the caller's org, so it could not show one that landed elsewhere) and requires the host's org; and a no-org control that leaves the resolver out. Red first: the org leg on the unchanged hosts failed at org alone (3 and 10 sessions on __fsd_default_org__). Green with the resolver. no-org fails at org alone; the existing controls still fail only their own legs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- .../goal.md | 6 ++++ .../host.mts | 12 ++++++- .../run.mts | 29 +++++++++++++++- .../a-routed-post-gets-one-answer/goal.md | 6 ++++ .../a-routed-post-gets-one-answer/host.mts | 14 ++++++-- .../a-routed-post-gets-one-answer/run.mts | 33 +++++++++++++++++-- 6 files changed, 94 insertions(+), 6 deletions(-) diff --git a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md index 7eeab09dda..fb062866af 100644 --- a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md +++ b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md @@ -15,6 +15,7 @@ - **answers**: the person's conversation holds exactly one answer line per post from each agent delegate, under its name, and none by anyone else. - **quiet**: no worker's conversation holds a delegate's answer as a turn it heard. - **other**: the `note` delegate holds no conversation. +- **org**: the person's conversation and every delegate conversation under it carry the org the host names in `resolvePrincipal`, not the development default (FIX-1792 BR-25). Read from the host's session store, since the router scopes what it lists to the caller's org and could not show a session that landed in another. - **source**: `host.mts` imports only `@flow-state-dev/*`, calls `defineCoordinatorFlow`, and builds no `dispatcher`, `keyedRouter` or `router`; the tree's coordinator routes to `everyone`. Kitchen-sink's `workforce/hire.ts` calls `defineCoordinatorFlow` and builds none of the three, and its `SEAT_ASKS` has no `wake` column. **Anti-game:** no assertion on the router's output, a dispatch handle, a routing record, or a unit test. Only what each worker's own conversation kept and what the person's conversation holds count, read through the routes, and only this run's tokens. The wait before grading polls until every request has settled and then gives a wrongly woken worker 1.5s to run; none of it is graded. Kitchen-sink's runtime half is not re-proved here: `kitchen-sink-talk`'s goals are re-run on the converted app. @@ -28,6 +29,8 @@ - `GOAL_CONTROL=no-wake`: the coordinator flow is registered with no flow a delegate takes a post on. Must FAIL at **woken** and **answers**, and at nothing else. - `GOAL_CONTROL=answers-go-on`: the coordinator's file is read with `rounds: 1`, so each round's answers go back out to the other delegates. Must FAIL at **answers** (each delegate answers the other's answer too) and **quiet**, and at nothing else. +- `GOAL_CONTROL=no-org`: the host's `resolvePrincipal` left out, so nothing names its org. Must FAIL at **org** (every session carries `__fsd_default_org__`), and at nothing else. + The **import** and **source** legs have no control: a checkout without the export fails **import**, and kitchen-sink's pre-conversion `workforce/hire.ts` fails **source**. Both were produced and are logged below. ## Verdict log @@ -46,3 +49,6 @@ The **import** and **source** legs have no control: a checkout without the expor | 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-wake`: **woken** and **answers** only — `desk.amy` and `desk.oz` hold 0 conversations, and 0 answer lines. | | 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=answers-go-on`: **answers** and **quiet** only — 4 answer lines each, and each heard the other's answer (*desk.oz, through desk.front: [reply:fresh-host] noted.*). | | 2026-10-09 | same | scripted | FAIL (by hand) | `defineCoordinatorFlow` dropped from the package barrel: failed at **import** only. Restored. | +| 2026-10-09 | `fix/FIX-1792-p1` on 343af0c77, uncommitted | scripted | FAIL | **The org leg added, the host not yet changed (FIX-1792 BR-25, from review).** **org** only: the conversation with `desk.front` and both delegate sessions carry `__fsd_default_org__`; the host had no `resolvePrincipal`, before the conversion and after it. Every other leg green. | +| 2026-10-09 | same | scripted | **PASS** | The host names `u_fresh_host` and `org_fresh_host` in `resolvePrincipal`. Every leg green; the conversation and its 2 delegate sessions carry `org_fresh_host`. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-org`: **org** only — the conversation and both delegate sessions carry `__fsd_default_org__`. `no-wake` (**woken**, **answers**) and `answers-go-on` (**answers**, **quiet**) re-taken: each its own legs only, **org** green. | diff --git a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/host.mts b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/host.mts index 1d76112095..4f59495360 100644 --- a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/host.mts +++ b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/host.mts @@ -12,7 +12,9 @@ * when asked and declare no delegated-post entry, so a post runs nothing on * them. Every other worker runs on the built-in `agent`, answered by a * scripted model so the check needs no key. Each flow is registered once; a - * delegate's conversation is a session on its flow, naming it. + * delegate's conversation is a session on its flow, naming it. The app names + * its one user and org in `resolvePrincipal`, so every session it makes, the + * delegates' included, carries that org rather than the development default. * * `seams` is the goal check's, for its controls, and nothing else. An app * passes nothing. @@ -57,6 +59,9 @@ const workerDoor = { /** The user who talks to the coordinator. */ export const OWNER = "u_fresh_host"; +/** The organization the app runs as. Every session it makes carries it. */ +export const ORG = "org_fresh_host"; + /** What an agent delegate answers with, whatever it heard. */ export const REPLY_MARKER = "[reply:fresh-host]"; @@ -84,6 +89,8 @@ export interface HostSeams { adaptWorkers?: (workers: WorkerManifest[]) => WorkerManifest[]; /** The flows a delegate takes a post on, given the ones the app names. */ adaptDelegateFlows?: (flows: { kind: string }[]) => { kind: string }[]; + /** Leave the app's `resolvePrincipal` out, so nothing names its org. */ + omitPrincipal?: boolean; } /** The app, booted: its router, and what it registered. */ @@ -122,6 +129,9 @@ export async function startFreshHost(tree: string, seams: HostSeams = {}): Promi const state = createFlowState({ flows: Object.fromEntries(copies.map((copy) => [copy.id, copy])), stores: { default: { primary: inMemoryStores() } }, + // Who every request is, and the org its sessions bind to. Ignores the + // request: one caller, and a body read would be caller-controlled identity. + ...(seams.omitPrincipal === true ? {} : { resolvePrincipal: () => ({ userId: OWNER, orgId: ORG }) }), modelResolver: createMockModelResolver({ generators: { "agent-answer": mockGenerator({ diff --git a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts index c088a6bbc3..d3c816b08d 100644 --- a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts +++ b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts @@ -20,6 +20,11 @@ * each agent delegate, under its name. * quiet no delegate hears another's answer. * other the delegate whose flow takes no posts holds nothing. + * org the person's conversation and every delegate conversation under + * it carry the host's named org, not the development default + * (FIX-1792 BR-25). Read from the store, since the router scopes + * what it lists to the caller's org and could not show one that + * landed elsewhere. * source the host imports only `@flow-state-dev/*`, registers the * coordinator flow and builds no dispatcher or router, and the * tree's coordinator routes to everyone; kitchen-sink registers @@ -32,6 +37,7 @@ * Run: pnpm --dir goals exec tsx workforce-mailboxes/a-fresh-host-wakes-its-member-agents/run.mts * Controls: GOAL_CONTROL=no-wake (no flow takes a delegated post: must FAIL at woken and answers, and nothing else) * GOAL_CONTROL=answers-go-on (the coordinator read with `rounds: 1`: must FAIL at answers and quiet, and nothing else) + * GOAL_CONTROL=no-org (the host's resolvePrincipal left out: must FAIL at org, and nothing else) */ import { randomUUID } from "node:crypto"; import { readFileSync } from "node:fs"; @@ -51,6 +57,8 @@ const EXPECTED: Record = { "no-wake": ["woken", "answers"], // Each round's answers go back out: each delegate hears the other's answer and answers it. "answers-go-on": ["answers", "quiet"], + // No resolver names the org: every session lands in the development default. + "no-org": ["org"], }; if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); @@ -66,6 +74,8 @@ function controlSeams(): HostSeams { adaptWorkers: (workers: WorkerManifest[]) => workers.map((w) => (w.declared.flow === "coordinator" ? { ...w, declared: { ...w.declared, rounds: 1 } } : w)), }; + case "no-org": + return { omitPrincipal: true }; default: return {}; } @@ -142,7 +152,7 @@ await runGoal(async (failures) => { ); } - const { startFreshHost, OWNER, REPLY_MARKER } = await import("./host.mts"); + const { startFreshHost, OWNER, ORG, REPLY_MARKER } = await import("./host.mts"); const app = await startFreshHost(TREE, controlSeams()); const call = async (method: "GET" | "POST", segments: string[], body?: unknown, query = "") => { const res = await app.router[method]( @@ -256,6 +266,23 @@ await runGoal(async (failures) => { if (held.length > 0) fail("other", `${seat} holds ${held.length} conversations (want 0)`); else evidence.push(`${seat}: no conversation`); } + + // ---- org: the conversation and each delegate's carry the named org --------- + { + const runtime = await app.state.getRuntime(); + const made = (await runtime.stores.session.list({ parentage: "all" })).filter( + (s) => s.id === conversation || s.parentSessionId === conversation, + ); + const delegateSessions = made.filter((s) => s.parentSessionId === conversation); + const off = made.filter((s) => s.orgId !== ORG); + // How many delegate sessions there should be is woken's to grade; this + // leg grades the org of the ones there are. + if (!made.some((s) => s.id === conversation)) fail("org", `the store holds no session ${conversation}`); + for (const s of off) fail("org", `${s.flowId ?? s.flowKind} session ${s.id} carries org ${String(s.orgId)} (want ${ORG})`); + if (!failures.some((f) => f.startsWith("[org]"))) { + evidence.push(`${conversation} and its ${delegateSessions.length} delegate session(s) carry ${ORG}`); + } + } } finally { await app.state.dispose(); } diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md index 73b128b3e2..9eacd8573d 100644 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md @@ -24,6 +24,7 @@ No goal proves either outcome on `main` after the conversion: the coordinator fl - **held**: a post sent while the device specialist has not answered its last post (its turn on that one replied with nothing, so no line) is heard by it alone, with no evaluation call. - **fallback**: a post the evaluation cannot place is heard by the fallback alone. - **everyone**: the post to the coordinator on `routing: everyone` is heard once by each of the four agents, and each one's answer lands in that conversation once. +- **org**: both of the person's conversations and every delegate conversation under them carry the org the host names in `resolvePrincipal`, not the development default (FIX-1792 BR-25). Read from the host's session store, since the router scopes what it lists to the caller's org and could not show a session that landed in another. **Anti-game:** no assertion on a routing record, a dispatch handle, a router decision, or a unit test. Who ran is read from each delegate's own kept conversation; what landed is read from the person's conversation; "no evaluation" is read from the route model's own calls, a real side effect. The scripts answer only from what they are handed: the route from the post (`[route:]`, else a failed call), the answer from its turn. Waiting is polling until every request in the host settles, ungraded. @@ -38,6 +39,8 @@ No goal proves either outcome on `main` after the conversion: the coordinator fl - `GOAL_CONTROL=no-route`: the best-fit coordinator's file read as `routing: everyone`. Every delegate hears every post and each answers it. Must FAIL at **one**, **lands**, **held** and **fallback**, and at nothing else. - `GOAL_CONTROL=no-landing`: the `agent` flow replaced by one of the app's own that takes a delegated post, answers it in its own conversation, and hands nothing back. Must FAIL at **lands**; because no specialist ever answers, the hold keeps each next post on the first specialist, so at **one** and **fallback** too; and at **everyone**, whose answers never land. Nothing else. +- `GOAL_CONTROL=no-org`: the host's `resolvePrincipal` left out, so nothing names its org. Must FAIL at **org** (every session carries `__fsd_default_org__`), and at nothing else. + The **import** and **source** legs have no control; each was reddened by hand, logged below. ## Verdict log @@ -67,3 +70,6 @@ The **import** and **source** legs have no control; each was reddened by hand, l | 2026-10-09 | same | scripted | FAIL (by hand) | The host given its own `dispatcher(…)`: failed at **source** only. Restored. | | 2026-10-09 | same | route `vercel/typesafe-ai/jev`, answers `vercel/openai/gpt-5.4-mini` | FAIL (live, follow-up pinned) | `GOAL_LIVE=1` with the follow-up still pinned to the laptop's specialist: the follow-up went to `support.accounts`, not `support.devices`; *Where can I buy it?* went to `support.general`, which answered about something else. The same finding as the faithful port, on real models. | | 2026-10-09 | same | as above | **PASS** (live) | `GOAL_LIVE=1`, the follow-up unpinned: each of the seven posts reached one delegate and landed one line under its name; the follow-up reached `support.accounts` again. | +| 2026-10-09 | `fix/FIX-1792-p1` on 343af0c77, uncommitted | scripted | FAIL | **The org leg added, the host not yet changed (FIX-1792 BR-25, from review).** **org** only: both conversations and all 8 delegate sessions carry `__fsd_default_org__`; the host had no `resolvePrincipal`, before the conversion and after it. Every other leg green. | +| 2026-10-09 | same | scripted | **PASS** | The host names `u_routed_host` and `org_routed_host` in `resolvePrincipal`. Every leg green; both conversations and their 8 delegate sessions carry `org_routed_host`. The live leg was not re-taken. | +| 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-org`: **org** only — both conversations and all 8 delegate sessions carry `__fsd_default_org__`. `no-route` (**one**, **lands**, **held**, **fallback**) and `no-landing` (**one**, **lands**, **fallback**, **everyone**) re-taken: each its own legs only, **org** green. | diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts index 41f09f9547..8d962d6415 100644 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts @@ -8,8 +8,10 @@ * Workforce's `coordinator` flow once, with the built-in `agent` as the flow * its delegates take posts on, and names the model best fit's one evaluator * call runs on. Every worker flow is registered once, and a delegate's - * conversation is a session on its flow, naming it. The goal check reads this - * file's source to hold it to that. + * conversation is a session on its flow, naming it. The app names its one + * user and org in `resolvePrincipal`, so every session it makes, the + * delegates' included, carries that org rather than the development default. + * The goal check reads this file's source to hold it to that. * * Keyless by default: the delegates answer from a scripted model, and best * fit's one evaluation is scripted by its block name, `coordinator-route`. @@ -47,6 +49,9 @@ import { readWorkforce } from "@flow-state-dev/workforce/loader"; /** The user who talks to the coordinators. */ export const OWNER = "u_routed_host"; +/** The organization the app runs as. Every session it makes carries it. */ +export const ORG = "org_routed_host"; + /** What every scripted answer carries. */ export const REPLY_MARKER = "[reply:routed-host]"; @@ -61,6 +66,8 @@ export interface HostSeams { agent?: (installation: WorkerInstallation) => { kind: string }; /** Real models: this resolver resolves the route's model and the delegates' answers. */ live?: { modelResolver: ModelResolver }; + /** Leave the app's `resolvePrincipal` out, so nothing names its org. */ + omitPrincipal?: boolean; } /** The app, booted. */ @@ -134,6 +141,9 @@ export async function startRoutedHost(tree: string, seams: HostSeams = {}): Prom const state = createFlowState({ flows: Object.fromEntries(copies.map((copy) => [copy.id, copy])), stores: { default: { primary: inMemoryStores() } }, + // Who every request is, and the org its sessions bind to. Ignores the + // request: one caller, and a body read would be caller-controlled identity. + ...(seams.omitPrincipal === true ? {} : { resolvePrincipal: () => ({ userId: OWNER, orgId: ORG }) }), modelResolver: seams.live?.modelResolver ?? createMockModelResolver({ diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts index bbdc032ec8..4794f81297 100644 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts @@ -26,6 +26,11 @@ * fallback a post the evaluation cannot place goes to the fallback alone. * everyone a post to the coordinator on `routing: everyone` reaches every * agent once, and each answer lands there once. + * org both of the person's conversations and every delegate + * conversation under them carry the host's named org, not the + * development default (FIX-1792 BR-25). Read from the store, since + * the router scopes what it lists to the caller's org and could not + * show one that landed elsewhere. * * With GOAL_LIVE=1 a live leg runs after them, on real models: the route on * the host's own evaluation model string, the answers on a small chat model. @@ -33,6 +38,7 @@ * Run: pnpm --dir goals exec tsx workforce-mailboxes/a-routed-post-gets-one-answer/run.mts * Controls: GOAL_CONTROL=no-route (the best-fit coordinator read as `routing: everyone`) * GOAL_CONTROL=no-landing (the agent flow replaced by one that answers a delegated post and hands nothing back) + * GOAL_CONTROL=no-org (the host's resolvePrincipal left out) * Live: GOAL_LIVE=1 (needs AI_GATEWAY_API_KEY) */ import { randomUUID } from "node:crypto"; @@ -65,7 +71,9 @@ const EXPECTED: Record = { "no-route": ["one", "lands", "held", "fallback"], // No answer ever lands, so the specialist on the first post holds every next // one: lands, every leg the hold then diverts, and the answers on everyone. - "no-landing": ["one", "lands", "fallback", "everyone"] + "no-landing": ["one", "lands", "fallback", "everyone"], + // No resolver names the org: every session lands in the development default. + "no-org": ["org"] }; if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); @@ -114,6 +122,8 @@ function controlSeams(): HostSeams { }; case "no-landing": return { agent: (installation: WorkerInstallation) => quietAgentFlow(installation) }; + case "no-org": + return { omitPrincipal: true }; default: return {}; } @@ -211,7 +221,7 @@ async function scriptedLegs(fail: (leg: string, line: string) => void, evidence: const [s1, s2, s3] = delegates.filter((m) => m !== fallback); if (s3 === undefined) throw new Error(`the routed coordinator needs three specialists beside its fallback; read ${delegates.join(", ")}`); - const { startRoutedHost, OWNER, REPLY_MARKER } = await import("./host.mts"); + const { startRoutedHost, OWNER, ORG, REPLY_MARKER } = await import("./host.mts"); const app = await startRoutedHost(TREE, controlSeams()); const io = reader(app, OWNER); const run = randomUUID().replace(/-/g, "").slice(0, 10); @@ -303,6 +313,25 @@ async function scriptedLegs(fail: (leg: string, line: string) => void, evidence: } if (ok) evidence.push(`${lounge.id}: ${agents.length} agents heard the post once, and each answer landed once`); } + + // ---- org: both conversations and each delegate's carry the named org ----- + { + const runtime = await app.state.getRuntime(); + const opened = [conversation, loungeConversation]; + const made = (await runtime.stores.session.list({ parentage: "all" })).filter( + (s) => opened.includes(s.id) || (s.parentSessionId != null && opened.includes(s.parentSessionId)) + ); + const delegateSessions = made.filter((s) => !opened.includes(s.id)); + // How many delegate sessions there should be is the other legs' to + // grade; this leg grades the org of the ones there are. + const missing = opened.filter((id) => !made.some((s) => s.id === id)); + const off = made.filter((s) => s.orgId !== ORG); + for (const id of missing) fail("org", `the store holds no session ${id}`); + for (const s of off) fail("org", `${s.flowId ?? s.flowKind} session ${s.id} carries org ${String(s.orgId)} (want ${ORG})`); + if (missing.length === 0 && off.length === 0) { + evidence.push(`both conversations and their ${delegateSessions.length} delegate session(s) carry ${ORG}`); + } + } } finally { await app.state.dispose(); } From 7a80a09a008ff0e89b9d66e9faf4a6b5b62ecf78 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 17:06:21 +0000 Subject: [PATCH 11/14] refactor(kitchen-sink): the no-landing stand-in loads its worker once per request (FIX-1792 P1) Folds Code Snob's shape proof (#2943, not merged). The stand-in built the same answer generator twice and wrapped each in a sequencer whose only job was to load the worker. It now loads the worker on request.onStarted, as the built-in agent does, and its two doors share one generator body. Same contract, checked before and after through the app's real config under no-landing: a delegated post is answered in the specialist's own session with its own instructions, nothing lands in the person's conversation, and a direct message to a specialist is answered. That check went red with the control off (the answer landed) and with the new onStarted removed (the delegate never answered). Off the control landingControl returns before building anything, so the built-in agent is untouched. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- apps/kitchen-sink/lib/landing-control.ts | 53 +++++++------------ .../goal.md | 1 + .../answers-a-clerk-note-or-files-it/goal.md | 1 + 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/apps/kitchen-sink/lib/landing-control.ts b/apps/kitchen-sink/lib/landing-control.ts index d971849526..aecb8adc46 100644 --- a/apps/kitchen-sink/lib/landing-control.ts +++ b/apps/kitchen-sink/lib/landing-control.ts @@ -18,7 +18,7 @@ * can never reach a deployed build. The published flow has no switch that * stops an answer landing, and must not. */ -import { defineFlow, generator, handler, sequencer, type BlockDefinition } from "@flow-state-dev/core"; +import { defineFlow, generator, handler, type BlockDefinition } from "@flow-state-dev/core"; import { AGENT_KIND, delegatedPostSchema, @@ -40,7 +40,7 @@ const heard = (post: DelegatedPost) => `${post.from}, through ${post.coordinator * @param catalog The tool catalog the real flow carries. A worker reaches the * entries its `tools:` names, as on the real flow. * @param installation The installation the real flow runs its workers on: - * this one does too, loading each turn's worker first, as the real flow does. + * this one does too, loading the turn's worker before the answer reads it. */ export function landingControl(catalog: Record>, installation: WorkerInstallation) { if (goalControl() !== "no-landing") return undefined; @@ -53,47 +53,34 @@ export function landingControl(catalog: Record (_input: unknown, ctx: { session: object }) => config(ctx).teamInstructions, (_input: unknown, ctx: { session: object }) => config(ctx).instructions, ]; - const answerPost = generator({ - name: "agent-answer", - inputSchema: delegatedPostSchema, - model: "intent/chat", - itemVisibility: { client: true, history: true }, - history: true, + const answer = { + name: "agent-answer" as const, + model: "intent/chat" as const, + itemVisibility: { client: true, history: true } as const, + history: true as const, prompt, - user: heard, tools, - }); - const answerTurn = generator({ - name: "agent-answer", - inputSchema: z.object({ message: z.string() }), - model: "intent/chat", - itemVisibility: { client: true, history: true }, - history: true, - prompt, - user: (input: { message: string }) => input.message, - tools, - }); - // The turn's worker, loaded before the answer reads its settings. - const resolve = (inputSchema: T) => - handler({ - name: "agent-resolve-worker", - inputSchema, - resources: { ...installation.resources }, - execute: async (_input, ctx) => { - await installation.resolveWorker(ctx, AGENT_KIND); - }, - }); + }; const turnInput = z.object({ message: z.string() }); + const loadWorker = handler({ + name: "agent-resolve-worker", + inputSchema: z.unknown(), + resources: { ...installation.resources }, + execute: async (_input, ctx) => { + await installation.resolveWorker(ctx, AGENT_KIND); + }, + }); return defineFlow({ kind: AGENT_KIND, cardinality: "collection", configSchema: settings, session: installation.session(), resources: { ...installation.resources }, + request: { onStarted: loadWorker }, actions: { run: { inputSchema: turnInput.strict(), - block: sequencer({ name: "agent-run", inputSchema: turnInput }).tap(resolve(turnInput)).step(answerTurn), + block: generator({ ...answer, inputSchema: turnInput, user: (input: { message: string }) => input.message }), userMessage: (input: { message: string }) => input.message, }, }, @@ -101,9 +88,7 @@ export function landingControl(catalog: Record actions: { onDelegatedPost: { inputSchema: delegatedPostSchema, - block: sequencer({ name: "agent-heard-post", inputSchema: delegatedPostSchema }) - .tap(resolve(delegatedPostSchema)) - .step(answerPost), + block: generator({ ...answer, inputSchema: delegatedPostSchema, user: heard }), userMessage: heard, }, }, diff --git a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md index f54e47e9e1..a9e74dd3b8 100644 --- a/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md +++ b/goals/kitchen-sink-talk/a-person-talks-to-a-seat-a-mailbox-and-back/goal.md @@ -64,3 +64,4 @@ | 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | FAIL (check defect) | **Rewritten**, first run. **b:only-its-own** read the conversation with `support.help` as `support.devices`' session: the panel still drew the coordinator conversation's messages when the rail marked the session current. The check now waits until the panel draws that session's own messages (by the server's read of it, not graded). `c1:setup` also red: "Talk" was pressed with a seat panel already open, so the composer was found before the conversation was. Leg c now starts from the coordinator's panel. And the worker flow's kind and its copy, both named `agent`, are now found by their own attributes, not by name. | | 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15, uncommitted | scripted | **PASS** | **Rewritten.** One fresh build. **Plain journey**, every assertion on the open page and after its reload: a, `support.devices` working +0.9s, its line +4.0s, cleared +4.0s, nobody else working; b, each seat working +0.9s before its one line at +3.9s to +4.0s, by `support.devices`, `support.accounts`, `support.general`, each specialist's session for the conversation holding only its own posts and `support.fsd` none; c1 and c2 at +0.1s, the recall naming C1 only, the conversation with `support.help` showing none of it; seg. **Controls**, each failing every assertion it names, no leg at setup, and its green legs green: `no-live` a:line, b:line (a:working, b:working also red, not counted); `no-landing` a:line, b:line (a:kept, b:working, b:alone, b:only-its-own, b:kept also red, not counted: with no answer landing, best fit holds every later post for `support.devices`); `no-route` a:line, a:alone, b:line, b:alone (a:kept, b:only-its-own, b:kept also red, not counted); `drop-user-message` c1:turn, c2:turn; `no-history` c2:recall. | | 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15 | route `vercel/typesafe-ai/jev`, the app's own answer models | **PASS** (smoke) | `GOAL_LIVE=1`. One line each within two minutes, only the answering specialist seen working, every run done before the next post: the wifi post by `support.devices` (2.9s); its follow-up by `support.accounts` (6.0s), which answered *I don't have enough context to diagnose this — I don't know … what "it" is* (the open question on the P1 PR); the phone by `support.devices` (2.9s), which called it *a separate issue from the wifi problem*; the double charge by `support.accounts` (3.9s); the FSD question by `support.fsd` (2.9s); the parking pass by `support.general` (2.9s). All six kept after the reload. | +| 2026-10-09 | `fix/FIX-1792-p1` on 4b597367f, uncommitted: the `no-landing` stand-in loads its worker once per request (folded from #2943) | scripted | **PASS** (control) | `GOAL_CONTROL=no-landing` re-taken after the fold, one fresh build: **a:line**, **b:line** (a:kept, b:working, b:alone, b:only-its-own, b:kept also red, not counted); **c1**, **c2** and **seg** green, so a direct conversation still answers. The same legs as before the fold. | diff --git a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md index e499f30f38..48d3b14a4f 100644 --- a/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md +++ b/goals/kitchen-sink-talk/answers-a-clerk-note-or-files-it/goal.md @@ -57,3 +57,4 @@ Since FIX-1788 the rail lists each flow, not each worker, so "four specialists n | 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-landing`: **answer** only — after the reload no line answers the token. **roster** green. | | 2026-10-09 | `fix/FIX-1792-p1` on 54fcd1b15 | scripted, then the app's own models | **PASS** | `GOAL_LIVE=1`, same build served out of test mode. Scripted legs as above. Live: the swollen battery answered by `support.devices`; the triple charge and the taken-over account by `support.accounts`; the wifi password by `support.devices`; the invoices by `support.accounts`. One answer line each. The taken-over account's answer says *I'm escalating this to our security/support team*; nothing escalates now. | | 2026-10-09 | `fix/FIX-1792-p1` at 482afa103, uncommitted: one sentence added to each support specialist's instructions, *Never promise to escalate a case or hand it to another team: answer what you can, and say plainly what you can't do.* | scripted, then the app's own models | **PASS** | `GOAL_LIVE=1`, re-taken for the row above, whose taken-over-account answer promised an escalation nothing makes. Scripted legs as above. Live: the swollen battery by `support.devices` (*I'm not able to arrange or guarantee a same-day replacement myself*); the triple charge by `support.accounts` (*I can't arrange for someone else to review this case*); the taken-over account by `support.accounts` (*I can't arrange for another person or team to do it either*, then the service's own recovery process); the wifi password by `support.devices`; the invoices by `support.accounts`. One answer line each, and none promises an escalation or a hand-off. | +| 2026-10-09 | `fix/FIX-1792-p1` on 4b597367f, uncommitted: the `no-landing` stand-in loads its worker once per request (folded from #2943) | scripted | FAIL (expected) | `GOAL_CONTROL=no-landing` re-taken after the fold: **answer** only — after the reload no line answers the token, as before the fold. **roster** green. | From d55a8c3209409da1bae006be44698bee9d003287 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 17:08:13 +0000 Subject: [PATCH 12/14] chore(specs): the FIX-1792 checker classifies goals/lib/shift-manager.mts (P1) FIX-1824 moved pendingSeatAsks into goals/lib/shift-manager.mts, whose doc comment says "a mailbox post". The word only: an EDIT row, FIX-1796's sweep. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- specs/issues/FIX-1792/poc/inventory/check.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/issues/FIX-1792/poc/inventory/check.mjs b/specs/issues/FIX-1792/poc/inventory/check.mjs index bae7aa0ba9..f3ac1abe77 100644 --- a/specs/issues/FIX-1792/poc/inventory/check.mjs +++ b/specs/issues/FIX-1792/poc/inventory/check.mjs @@ -472,6 +472,7 @@ const FILE_CLASS = { const GOALS = { "goals/README.md": "EDIT · the word, FIX-1796", "goals/lib/workers.mts": "EDIT · the hire's mailboxBoards option goes with S9", + "goals/lib/shift-manager.mts": "EDIT · the word, in pendingSeatAsks's doc comment (FIX-1824's move); FIX-1796", "goals/coordinators/hands-each-post-to-its-delegates": "CONVERT · FIX-1791's goal check reads the DevTeam's mailbox boards; it follows the DevTeam in P3 (S7)", "goals/agent-discovery/an-orchestrator-routes-a-task-by-asking": "EDIT · a roster literal's mailboxes field", "goals/design-system/skins-reused-components-from-one-token-set": "EDIT · a rail label in a demo host", From 419d5aca6fa5a94b7e12a1a5c54827bf2f7faac9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 00:14:16 +0000 Subject: [PATCH 13/14] chore(specs): the FIX-1792 checker classifies two files main added (P1) FIX-1794 P2's conversation-board task entry takes mailbox task lists (an S9 edit), and FIX-1776's chief-of-staff memory test passes the DevTeam host its mailboxes option (an S7 conversion, as its coordinator sibling is). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- specs/issues/FIX-1792/poc/inventory/check.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/issues/FIX-1792/poc/inventory/check.mjs b/specs/issues/FIX-1792/poc/inventory/check.mjs index f3ac1abe77..795a70f305 100644 --- a/specs/issues/FIX-1792/poc/inventory/check.mjs +++ b/specs/issues/FIX-1792/poc/inventory/check.mjs @@ -272,6 +272,7 @@ const FILE_CLASS = { "packages/workforce/src/best-fit.ts": "E · S9 · a comment names the mailbox's route wrapper, routeByPurpose", "packages/shift-manager/test/devteam-cos-tool-names.test.ts": "E · S7 · the chief of staff loses setWorkstreams", "packages/shift-manager/test/devteam-chief-of-staff-coordinator.test.ts": "C · S7 · the DevTeam host's mailboxes option in a test; the host moves off mailboxes", + "packages/shift-manager/test/devteam-chief-of-staff-memory.test.ts": "C · S7 · the DevTeam host's mailboxes option in a test; the host moves off mailboxes", ".agents/skills/concept-doc/example/CONCEPT.md": "E · S12 · the concept-doc skill's worked example names removed exports; P4 edits it or `--after` fails on it", ".omp/extensions/mailbox.ts": "U · — · the agent-mailbox tool's command, not Workforce", "apps/docs/docs/client/react.md": "E · S12 · an example opens a session on flowKind \"mailbox\"", @@ -385,6 +386,7 @@ const FILE_CLASS = { "packages/ui/test/substrate-components-never-raw.test.ts": "E · S9 · tests", "packages/workforce/README.md": "E · S12 · the mailbox floor's exports", "packages/workforce/src/agent-worker-flow.ts": "E · S9 · taskLists from mailbox boards go", + "packages/workforce/src/conversation-board/task-entry.ts": "E · S9 · its mailboxLists option and the mailbox board ledgers go (FIX-1794 P2's task entry)", "packages/workforce/src/browser.ts": "E · S9 · browser exports", "packages/workforce/src/codegen/discover.ts": "E · S2 · the flows/mailboxes slot goes", "packages/workforce/src/codegen/index.ts": "E · S2 · the slot list", From 80e4f6ad1e25113e2521d516a80ef04f3c08a84a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 00:14:31 +0000 Subject: [PATCH 14/14] feat(goals): the routed-post goal's followup and context legs are back (FIX-1792 P1) FIX-1828 gave best fit the conversation's recent lines ({ recent, post }) and hands a delegate those lines as one user-role message before the post. So the two legs the conversion retired are restored: the scripted route sends a [follow-up] to whoever last wrote in recent, and the scripted answer reads the earlier item from the lines message, never system text. Each has a control wired as the others are: no-recent (best fit's evaluation handed { post } alone, around the model) fails at followup only; no-context (an agent flow of the app's own whose delegated post reaches the published entry without its lines) fails at context only. no-route and no-landing redden both again, as they did on the mailbox. The live leg pins the follow-up to the laptop's specialist again and strips the container's intent overrides. Verdicts in each goal.md, fresh-host re-taken after the merge. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XMg2dz8sMEdyzCQTRsaMTV --- .../goal.md | 1 + .../a-routed-post-gets-one-answer/goal.md | 27 ++-- .../a-routed-post-gets-one-answer/host.mts | 37 ++++-- .../a-routed-post-gets-one-answer/run.mts | 119 ++++++++++++++++-- 4 files changed, 157 insertions(+), 27 deletions(-) diff --git a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md index fb062866af..a699a6c197 100644 --- a/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md +++ b/goals/workforce-mailboxes/a-fresh-host-wakes-its-member-agents/goal.md @@ -52,3 +52,4 @@ The **import** and **source** legs have no control: a checkout without the expor | 2026-10-09 | `fix/FIX-1792-p1` on 343af0c77, uncommitted | scripted | FAIL | **The org leg added, the host not yet changed (FIX-1792 BR-25, from review).** **org** only: the conversation with `desk.front` and both delegate sessions carry `__fsd_default_org__`; the host had no `resolvePrincipal`, before the conversion and after it. Every other leg green. | | 2026-10-09 | same | scripted | **PASS** | The host names `u_fresh_host` and `org_fresh_host` in `resolvePrincipal`. Every leg green; the conversation and its 2 delegate sessions carry `org_fresh_host`. | | 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-org`: **org** only — the conversation and both delegate sessions carry `__fsd_default_org__`. `no-wake` (**woken**, **answers**) and `answers-go-on` (**answers**, **quiet**) re-taken: each its own legs only, **org** green. | +| 2026-10-10 | `fix/FIX-1792-p1` on 4b54c9df9 (`main` at 30edb4cb7 merged), uncommitted | scripted | **PASS** | Re-taken after the merge (the coordinator now carries a conversation's recent lines to best fit and its delegates). Every leg green. `no-wake` (**woken**, **answers**), `answers-go-on` (**answers**, **quiet**) and `no-org` (**org**) each failed at its own legs only. | diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md index 9eacd8573d..d7a0979ebd 100644 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/goal.md @@ -2,16 +2,14 @@ **Issue:** FIX-1610 (VG of the spec's PLAN; feeds the epic FIX-1592's closure, FIX-1601). Converted by FIX-1792 (BR-8, BR-9): the two mailboxes are coordinators' worker files now. -**Outcome:** A person's post to a coordinator on `routing: best-fit` runs exactly one specialist, the one its purpose points to, and that specialist's answer lands in the person's conversation as its own line every time. A post sent while that specialist has not answered yet goes to it too, with no model call. A coordinator on `routing: everyone` hands each post to every delegate. The host writes no router: it registers Workforce's `coordinator` flow and names the model best fit's one call runs on. +**Outcome:** A person's post to a coordinator on `routing: best-fit` runs exactly one specialist, the one its purpose and the conversation's recent lines point to; that specialist answers with the recent lines in view, and its answer lands in the person's conversation as its own line every time. A post sent while that specialist has not answered yet goes to it too, with no model call. A coordinator on `routing: everyone` hands each post to every delegate. The host writes no router: it registers Workforce's `coordinator` flow and names the model best fit's one call runs on. **Converted (FIX-1792).** The folder name is kept so its verdict history stays in one place. Until FIX-1792 the routed mailbox was a `MAILBOX.md` with `routing: fallback:`, routed by `routeByPurpose`, and the unrouted one woke every member through `wakeMemberSeats`. They are coordinators now, key by key: `members:` became `delegates:`, `routing: { fallback: x }` became `routing: best-fit` and `fallback: x`, and the file with no `routing:` line got `routing: everyone`. The **unrouted** leg is now **everyone**: on a coordinator each delegate's answer lands in the conversation, where the unrouted mailbox kept none. -**Retired legs.** Two legs, and the two controls that served only them, have no subject on the coordinator flow, and are retired here under FIX-1792 BR-24 with the run that shows it (logged below): +**Restored legs.** The conversion first retired two legs, because the coordinator flow (FIX-1791) routed a post on its own words and showed a delegate no other delegate's lines. FIX-1828 gave the coordinator both, and the legs are back, each with a control: -- **followup** (a follow-up after an answer is routed by an evaluation that saw that answer's line) and its control `no-transcript`. The coordinator's best fit routes a post on its own words: its one evaluation is handed the post, not the conversation's recent lines. A follow-up sent while the specialist is still on the last post is still held for it (**held**); one sent after the answer is placed on its words alone. Live, "It sees it. It fails right after the password." went to `support.accounts`, where the mailbox sent it to `support.devices`. -- **context** ("where can I buy it?" answered by a specialist never sent the post naming "it", from that post's line) and its control `no-context`. A delegate's turn is handed the post it is delivered, and its own earlier deliveries from this conversation; never another delegate's lines. - -No goal proves either outcome on `main` after the conversion: the coordinator flow (FIX-1791) does not offer it. Whether it should is put to the product owner on FIX-1792's P1 PR. +- **followup** (a follow-up after an answer is routed by an evaluation that saw that answer's line). Best fit's evaluation state is `{ recent, post }`, each line `{ from, text }`, so the scripted route reads it as it read the mailbox's. Control `no-recent` (the mailbox's `no-transcript`). +- **context** ("where can I buy it?" answered by a specialist never sent the post naming "it", from that post's line). The delegate that takes a post sees the recent lines as one user-role message just before the post, never as system text, and its stored conversation never keeps them. Control `no-context`. **Input:** `fixtures/workforce/`: one team, four agent specialists (no `flow:`, each with a `description:`), one coordinator on `routing: best-fit` with a `fallback:` naming one of them, and one on `routing: everyone` naming all four. `host.mts` is the app: the tree, the published packages, and the `coordinator` flow registered with the built-in `agent` as the flow its delegates take posts on. Held-out: the specialists and the fallback are read off the tree (the routed coordinator's `delegates:` and `fallback:`), never hardcoded, and each run's posts carry fresh tokens. @@ -21,23 +19,27 @@ No goal proves either outcome on `main` after the conversion: the coordinator fl - **source**: `host.mts` imports only `@flow-state-dev/*`, calls `defineCoordinatorFlow`, and builds no `dispatcher`, `keyedRouter` or `router`; exactly one coordinator in the tree declares `routing: best-fit`. - **one**: a device, an account and a framework post are each heard by their specialist and by no other delegate. - **lands**: each of those, and the unclear post, has exactly one answer line in the conversation, by the specialist that was routed it. +- **followup**: a follow-up marked `[follow-up]`, sent after the account specialist's answer landed, is heard by that specialist alone, placed by one evaluation call whose `recent` holds a line by it. - **held**: a post sent while the device specialist has not answered its last post (its turn on that one replied with nothing, so no line) is heard by it alone, with no evaluation call. +- **context**: "where can I buy it?", routed to the fallback (never sent the laptop post naming `item-…`), is answered naming that item, read from the lines message just before the post. The fallback's stored conversation keeps neither that line nor any message carrying another post's token. - **fallback**: a post the evaluation cannot place is heard by the fallback alone. - **everyone**: the post to the coordinator on `routing: everyone` is heard once by each of the four agents, and each one's answer lands in that conversation once. - **org**: both of the person's conversations and every delegate conversation under them carry the org the host names in `resolvePrincipal`, not the development default (FIX-1792 BR-25). Read from the host's session store, since the router scopes what it lists to the caller's org and could not show a session that landed in another. -**Anti-game:** no assertion on a routing record, a dispatch handle, a router decision, or a unit test. Who ran is read from each delegate's own kept conversation; what landed is read from the person's conversation; "no evaluation" is read from the route model's own calls, a real side effect. The scripts answer only from what they are handed: the route from the post (`[route:]`, else a failed call), the answer from its turn. Waiting is polling until every request in the host settles, ungraded. +**Anti-game:** no assertion on a routing record, a dispatch handle, a router decision, or a unit test. Who ran is read from each delegate's own kept conversation; what landed is read from the person's conversation; "no evaluation" is read from the route model's own calls, a real side effect. The scripts answer only from what they are handed: the route from the post (`[route:]`; `[follow-up]` to whoever last wrote in the `recent` lines beside it; else a failed call), the answer from its turn (`[answer:from-context]` from the user-role lines message just before it, never the system text). Waiting is polling until every request in the host settles, ungraded. **Model:** scripted, keyless (epic FIX-1592 D3), for every graded leg. The live leg (below) runs once with a key: the route on the host's own model string `vercel/typesafe-ai/jev` through the app's resolver, the answers on `vercel/openai/gpt-5.4-mini`. **Run:** `pnpm --dir goals exec tsx workforce-mailboxes/a-routed-post-gets-one-answer/run.mts` -**Live:** `GOAL_LIVE=1` on the same command, with `AI_GATEWAY_API_KEY`. After the scripted legs, the same host on real models: the POC's five posts, the laptop post's follow-up, and "where can I buy it?". Each post must reach one delegate (the expected one where the POC named it) and get one line by it. The follow-up names no expected delegate since the retirement above. +**Live:** `GOAL_LIVE=1` on the same command, with `AI_GATEWAY_API_KEY`. After the scripted legs, the same host on real models: the POC's five posts, the laptop post's follow-up, and "where can I buy it?". Each post must reach one delegate (the expected one where the POC named it; the follow-up's is the laptop's specialist) and get one line by it. The run strips the container's `FSDEV_DEFAULT_MODEL` / `FSDEV_INTENT_*`, so the answers run on the model named here. **Controls:** on the same command. Each is applied by the check around the host, never inside a package. -- `GOAL_CONTROL=no-route`: the best-fit coordinator's file read as `routing: everyone`. Every delegate hears every post and each answers it. Must FAIL at **one**, **lands**, **held** and **fallback**, and at nothing else. -- `GOAL_CONTROL=no-landing`: the `agent` flow replaced by one of the app's own that takes a delegated post, answers it in its own conversation, and hands nothing back. Must FAIL at **lands**; because no specialist ever answers, the hold keeps each next post on the first specialist, so at **one** and **fallback** too; and at **everyone**, whose answers never land. Nothing else. +- `GOAL_CONTROL=no-route`: the best-fit coordinator's file read as `routing: everyone`. Every delegate hears every post and each answers it. Must FAIL at **one**, **lands**, **followup**, **held**, **context** and **fallback**, and at nothing else. +- `GOAL_CONTROL=no-landing`: the `agent` flow replaced by one of the app's own that takes a delegated post, answers it in its own conversation, and hands nothing back. Must FAIL at **lands**; because no specialist ever answers, the hold keeps each next post on the first specialist, so at **one**, **followup**, **context** and **fallback** too; and at **everyone**, whose answers never land. Nothing else. +- `GOAL_CONTROL=no-recent`: best fit's scripted evaluation handed `{ post }` alone, by wrapping the model the host routes with. Must FAIL at **followup** only. +- `GOAL_CONTROL=no-context`: the `agent` flow replaced by one of the app's own that takes a delegated post through the published `delegatedPostEntry`, with `delegatedPostHistory`, and lands its answer, but strips the post's lines before the entry. Must FAIL at **context** only. - `GOAL_CONTROL=no-org`: the host's `resolvePrincipal` left out, so nothing names its org. Must FAIL at **org** (every session carries `__fsd_default_org__`), and at nothing else. @@ -73,3 +75,8 @@ The **import** and **source** legs have no control; each was reddened by hand, l | 2026-10-09 | `fix/FIX-1792-p1` on 343af0c77, uncommitted | scripted | FAIL | **The org leg added, the host not yet changed (FIX-1792 BR-25, from review).** **org** only: both conversations and all 8 delegate sessions carry `__fsd_default_org__`; the host had no `resolvePrincipal`, before the conversion and after it. Every other leg green. | | 2026-10-09 | same | scripted | **PASS** | The host names `u_routed_host` and `org_routed_host` in `resolvePrincipal`. Every leg green; both conversations and their 8 delegate sessions carry `org_routed_host`. The live leg was not re-taken. | | 2026-10-09 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-org`: **org** only — both conversations and all 8 delegate sessions carry `__fsd_default_org__`. `no-route` (**one**, **lands**, **held**, **fallback**) and `no-landing` (**one**, **lands**, **fallback**, **everyone**) re-taken: each its own legs only, **org** green. | +| 2026-10-10 | `fix/FIX-1792-p1` on 4b54c9df9 (`main` at 30edb4cb7 merged, FIX-1828 in), uncommitted | scripted | **PASS** | **followup and context restored.** The follow-up reached `support.accounts` by one evaluation whose `recent` held its line; `support.general`, never sent the laptop post, answered *You can buy the item-… at the shop.* from the user-role lines message before the post, and kept none of the lines. Every other leg green as before; both conversations and their 8 delegate sessions carry `org_routed_host`. | +| 2026-10-10 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-recent`: **followup** only — `tok-…p3` heard by `support.general` (want `support.accounts`); its one evaluation call saw no line by `support.accounts`. | +| 2026-10-10 | same | scripted | FAIL (expected) | `GOAL_CONTROL=no-context`: **context** only — `support.general` answered *Buy what?*. | +| 2026-10-10 | same | scripted | FAIL (expected) | `no-route` re-taken: **one**, **lands**, **followup**, **held**, **context**, **fallback** only (context: the fallback, sent every post, keeps the laptop line). `no-landing` re-taken: **one**, **lands**, **followup**, **context**, **fallback**, **everyone** only (the hold keeps the follow-up and the buy-it post on `support.devices`); **held** green. `no-org`: **org** only. | +| 2026-10-10 | same | route `vercel/typesafe-ai/jev`, answers `vercel/openai/gpt-5.4-mini` | **PASS** (live) | `GOAL_LIVE=1`, attempt 1 of 1, the follow-up pinned again: "It sees it. It fails right after the password." → `support.devices` (answered about forgetting the office Wi-Fi and re-entering the password); "Where can I buy it?" → `support.devices` (*If you mean the cable…*); the account, framework and parking posts to `support.accounts`, `support.fsd`, `support.general`. One line each, by the delegate that heard it. | diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts index 8d962d6415..cbf374fabc 100644 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/host.mts @@ -18,11 +18,15 @@ * The scripts are the app's test doubles, and they answer from what they are * handed: * - * - the route: a post naming `[route:]` goes to that delegate; - * anything else fails the call, so the coordinator's fallback takes it. + * - the route: a post naming `[route:]` goes to that delegate; one + * marked `[follow-up]` goes to whoever last wrote in the recent lines it is + * handed beside the post, other than the post's writer; anything else fails + * the call, so the coordinator's fallback takes it. * - an answer: `[answer:none]` replies with nothing, which is a turn with no - * answer; anything else acknowledges the post's `tok-…`. Every reply carries - * {@link REPLY_MARKER}. + * answer; `[answer:from-context]` names the `item-…` it finds in the + * conversation's lines, the user-role message just before the post (never + * the system text); anything else acknowledges the post's `tok-…`. Every + * reply carries {@link REPLY_MARKER}. * * `seams` is the goal check's, for its controls and its live leg, and nothing * else. An app passes nothing. @@ -62,6 +66,8 @@ export const ROUTE_MODEL = "vercel/typesafe-ai/jev"; export interface HostSeams { /** The worker files as the host reads them, before anything is built. */ adaptWorkers?: (workers: WorkerManifest[]) => WorkerManifest[]; + /** Best fit's scripted evaluation, wrapped before the host routes with it. */ + adaptRoute?: (model: EvaluationModel) => EvaluationModel; /** The `agent` flow, replaced by one of the app's own built on the installation. */ agent?: (installation: WorkerInstallation) => { kind: string }; /** Real models: this resolver resolves the route's model and the delegates' answers. */ @@ -85,13 +91,20 @@ function textOf(content: unknown): string { return Array.isArray(content) ? content.map((part) => (part as { text?: string }).text ?? "").join("") : ""; } -/** Best fit's scripted evaluation. Reads the `{ post }` state the coordinator hands it. */ +/** Best fit's scripted evaluation. Reads the `{ recent, post }` state the coordinator hands it. */ function scriptedRoute() { return mockEvaluationModel({ answers: ({ state }) => { - const { post } = state as { post: { from: string; text: string } }; + const { recent = [], post } = state as { + recent?: Array<{ from: string; text: string }>; + post: { from: string; text: string }; + }; const named = /\[route:([^\]\s]+)\]/.exec(post.text)?.[1]; if (named !== undefined) return { member: { type: "choice", choice: named } }; + if (post.text.includes("[follow-up]")) { + const last = [...recent].reverse().find((line) => line.from !== post.from); + if (last !== undefined) return { member: { type: "choice", choice: last.from } }; + } throw new Error("the scripted route has no delegate for this post"); } }); @@ -105,9 +118,17 @@ function scriptedAnswer(): MockGeneratorInstance { reset: () => {}, next: (input: unknown): MockGeneratorScriptStep => { const messages = input as Array<{ role: string; content: unknown }>; - const turn = textOf([...messages].reverse().find((m) => m.role === "user")?.content); + const at = messages.map((m) => m.role).lastIndexOf("user"); + const turn = textOf(messages[at]?.content); if (turn.includes("[answer:none]")) return { text: "" }; const token = /tok-[a-z0-9]+/.exec(turn)?.[0] ?? ""; + if (turn.includes("[answer:from-context]")) { + // The conversation's lines come as one user-role message just before the post. + const before = messages[at - 1]; + const lines = before?.role === "user" ? textOf(before.content) : ""; + const item = /item-[a-z0-9]+/.exec(lines)?.[0]; + return { text: `${REPLY_MARKER} ${token} ${item === undefined ? "Buy what?" : `You can buy the ${item} at the shop.`}` }; + } return { text: `${REPLY_MARKER} ${token} noted.` }; } }; @@ -148,7 +169,7 @@ export async function startRoutedHost(tree: string, seams: HostSeams = {}): Prom seams.live?.modelResolver ?? createMockModelResolver({ generators: { "agent-answer": scriptedAnswer() }, - evaluators: { "coordinator-route": route as EvaluationModel }, + evaluators: { "coordinator-route": seams.adaptRoute?.(route as EvaluationModel) ?? (route as EvaluationModel) }, policy: "allow" }) } as never); diff --git a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts index 4794f81297..a655879988 100644 --- a/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts +++ b/goals/workforce-mailboxes/a-routed-post-gets-one-answer/run.mts @@ -21,8 +21,13 @@ * specialist, and nobody else. * lands each of those, and the unclear post, gets exactly one line in * the conversation, by the specialist that answered it. + * followup a follow-up after the account answer reaches the account + * specialist, by an evaluation that saw that answer's line. * held a post sent while the specialist has not answered yet goes to * it alone, with no evaluation. + * context "where can I buy it?" is answered, by a specialist that was + * never sent the post naming "it", from that post's line; its + * stored conversation keeps only its own posts and answers. * fallback a post the evaluation cannot place goes to the fallback alone. * everyone a post to the coordinator on `routing: everyone` reaches every * agent once, and each answer lands there once. @@ -39,6 +44,8 @@ * Controls: GOAL_CONTROL=no-route (the best-fit coordinator read as `routing: everyone`) * GOAL_CONTROL=no-landing (the agent flow replaced by one that answers a delegated post and hands nothing back) * GOAL_CONTROL=no-org (the host's resolvePrincipal left out) + * GOAL_CONTROL=no-recent (best fit's evaluation handed the post without the conversation's lines) + * GOAL_CONTROL=no-context (the agent flow replaced by one whose delegated post arrives without its lines) * Live: GOAL_LIVE=1 (needs AI_GATEWAY_API_KEY) */ import { randomUUID } from "node:crypto"; @@ -47,8 +54,10 @@ import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; import { z } from "zod"; import { createModelResolver, defineFlow, generator, handler } from "@flow-state-dev/core"; -import type { ModelResolver } from "@flow-state-dev/core/types"; +import type { EvaluationModel, ModelResolver } from "@flow-state-dev/core/types"; import { + delegatedPostEntry, + delegatedPostHistory, delegatedPostSchema, workerConfigOf, workerConfigSchema, @@ -57,7 +66,7 @@ import { type WorkerManifest } from "@flow-state-dev/workforce"; import { readWorkforce } from "@flow-state-dev/workforce/loader"; -import { gatewayModel, runGoal } from "../../lib/index.mts"; +import { gatewayModel, runGoal, stripIntentOverrides } from "../../lib/index.mts"; import type { HostSeams, RoutedHost } from "./host.mts"; const TREE = fileURLToPath(new URL("./fixtures/workforce", import.meta.url)); @@ -68,12 +77,16 @@ const LIVE = process.env.GOAL_LIVE === "1"; /** The legs each control must redden, and only those. Set from the runs logged in goal.md. */ const EXPECTED: Record = { // Every delegate hears every post, and each answers it: every best-fit leg. - "no-route": ["one", "lands", "held", "fallback"], + "no-route": ["one", "lands", "followup", "held", "context", "fallback"], // No answer ever lands, so the specialist on the first post holds every next // one: lands, every leg the hold then diverts, and the answers on everyone. - "no-landing": ["one", "lands", "fallback", "everyone"], + "no-landing": ["one", "lands", "followup", "context", "fallback", "everyone"], // No resolver names the org: every session lands in the development default. - "no-org": ["org"] + "no-org": ["org"], + // The evaluation cannot see the account answer, so the follow-up falls back. + "no-recent": ["followup"], + // The delegate's turn is shown no lines, so "it" has no referent. + "no-context": ["context"] }; if (CONTROL !== "" && EXPECTED[CONTROL] === undefined) { throw new Error(`unknown GOAL_CONTROL "${CONTROL}"; known: ${Object.keys(EXPECTED).join(", ")}`); @@ -112,6 +125,43 @@ function quietAgentFlow(installation: WorkerInstallation) { } as never); } +/** + * `no-context`'s agent flow: takes a delegated post through the published + * entry, answers it through `agent-answer` with `delegatedPostHistory`, and + * hands the answer back, as the built-in one does; but the post reaches the + * entry with its lines stripped, so the turn is shown none. + */ +function linelessAgentFlow(installation: WorkerInstallation) { + const run = generator({ + name: "agent-answer", + inputSchema: z.object({ message: z.string() }), + model: "scripted/answer", + prompt: (_input: { message: string }, ctx) => (workerConfigOf(ctx) as { instructions?: string }).instructions ?? "", + history: delegatedPostHistory, + user: (i: { message: string }) => i.message + }); + const entry = delegatedPostEntry(run); + const loadWorker = handler({ + name: "lineless-agent-load-worker", + inputSchema: z.unknown(), + resources: { ...installation.resources }, + execute: async (_input, ctx) => ({ worker: (await installation.resolveWorker(ctx, "agent")).id }) + }); + return defineFlow({ + kind: "agent", + configSchema: workerConfigSchema(), + session: installation.session(), + resources: { ...installation.resources }, + request: { onStarted: loadWorker }, + actions: { run: { inputSchema: z.object({ message: z.string() }), block: run, userMessage: (i: { message: string }) => i.message } }, + internal: { + actions: { + onDelegatedPost: { ...entry, block: entry.block.connectInput(({ recent: _recent, ...post }: DelegatedPost) => post) } + } + } + } as never); +} + /** The control's seams on the host, or none. */ function controlSeams(): HostSeams { switch (CONTROL) { @@ -124,6 +174,19 @@ function controlSeams(): HostSeams { return { agent: (installation: WorkerInstallation) => quietAgentFlow(installation) }; case "no-org": return { omitPrincipal: true }; + case "no-recent": + return { + adaptRoute: (model: EvaluationModel) => + ({ + ...model, + doEvaluate: (call: { state: { recent?: unknown } }) => { + const { recent: _recent, ...state } = call.state; + return (model as unknown as { doEvaluate: (c: unknown) => unknown }).doEvaluate({ ...call, state }); + } + }) as unknown as EvaluationModel + }; + case "no-context": + return { agent: (installation: WorkerInstallation) => linelessAgentFlow(installation) }; default: return {}; } @@ -226,13 +289,16 @@ async function scriptedLegs(fail: (leg: string, line: string) => void, evidence: const io = reader(app, OWNER); const run = randomUUID().replace(/-/g, "").slice(0, 10); const tok = (n: number) => `tok-${run}p${n}`; + const item = `item-${run}`; const posts = { device: `[route:${s1}] ${tok(1)} My phone stopped charging, and it's a brand new cable.`, account: `[route:${s2}] ${tok(2)} Different thing: I was charged twice for my subscription this month.`, + followup: `[follow-up] ${tok(3)} It was the visa card.`, framework: `[route:${s3}] ${tok(4)} How do I make a generator return structured output?`, unclear: `${tok(5)} Who do I ask about getting a parking pass?`, - unanswered: `[route:${s1}] [answer:none] ${tok(6)} My laptop won't join the office wifi.`, + unanswered: `[route:${s1}] [answer:none] ${tok(6)} My laptop ${item} won't join the office wifi.`, held: `${tok(7)} It sees the network. It fails right after the password.`, + buyIt: `[route:${fallback}] [answer:from-context] ${tok(8)} Where can I buy it?`, lounge: `${tok(9)} Morning, all.` }; @@ -250,7 +316,7 @@ async function scriptedLegs(fail: (leg: string, line: string) => void, evidence: const callsFor = (token: string) => app.routeCalls.filter((c) => (c.state as { post: { text: string } }).post.text.includes(token)); - for (const body of [posts.device, posts.account, posts.framework, posts.unclear, posts.unanswered, posts.held]) { + for (const body of [posts.device, posts.account, posts.followup, posts.framework, posts.unclear, posts.unanswered, posts.held, posts.buyIt]) { await io.post(conversation, body); await quiet(app, 8_000); } @@ -274,6 +340,16 @@ async function scriptedLegs(fail: (leg: string, line: string) => void, evidence: } else evidence.push(`${token}'s answer landed once, by ${want}`); } + // ---- followup: to the account specialist, by an evaluation that saw its line + { + const who = await heardBy(tok(3)); + const calls = callsFor(tok(3)); + const saw = calls.some((c) => ((c.state as { recent?: Array<{ from: string }> }).recent ?? []).some((l) => l.from === s2)); + if (who.length !== 1 || who[0] !== s2) fail("followup", `${tok(3)} was heard by [${who.join(", ")}] (want [${s2}])`); + if (calls.length !== 1 || !saw) fail("followup", `${tok(3)}'s route made ${calls.length} evaluation call(s), ${saw ? "seeing" : "not seeing"} a line by ${s2} (want one, seeing it)`); + if (who.length === 1 && who[0] === s2 && calls.length === 1 && saw) evidence.push(`the follow-up reached ${s2} by one evaluation that saw its line`); + } + // ---- held: sent before the specialist answered, so no evaluation -------- { const unanswered = lines.filter((l) => l.author !== undefined && l.text.includes(tok(6))); @@ -285,6 +361,30 @@ async function scriptedLegs(fail: (leg: string, line: string) => void, evidence: if (unanswered.length === 0 && who.length === 1 && who[0] === s1 && calls.length === 0) evidence.push(`${tok(7)} went to ${s1}, still on the last post, with no evaluation`); } + // ---- context: answered from a line the answering delegate was never sent -- + { + let ok = true; + const answer = lines.filter((l) => l.author === fallback && l.text.includes(tok(8))); + if (answer.length !== 1 || !answer[0]!.text.includes(item)) { + ok = false; + fail("context", `${fallback}'s answer to ${tok(8)} does not name ${item}: ${JSON.stringify(answer)}`); + } + // The post that named "it", as the conversation holds it: in no message the delegate kept. + const named = lines.find((l) => l.author === undefined && l.text.includes(tok(6)))?.text ?? posts.unanswered; + const kept = (await io.conversationsOf(fallback, conversation)).flat(); + if (kept.some((m) => m.text.includes(named))) { + ok = false; + fail("context", `${fallback}'s stored conversation keeps the line it was shown as context`); + } + // Each kept message is about one post: its own. The lines it was shown are in none. + const crowded = kept.filter((m) => new Set(m.text.match(/tok-[a-z0-9]+/g) ?? []).size > 1); + if (crowded.length > 0) { + ok = false; + fail("context", `${fallback} kept message(s) carrying other posts' lines: ${JSON.stringify(crowded)}`); + } + if (ok) evidence.push(`${fallback}, never sent ${tok(6)}, answered "${answer[0]!.text}", and kept none of the lines it was shown`); + } + // ---- fallback: a post nobody could place goes to the fallback alone ------ { const who = await heardBy(tok(5)); @@ -344,6 +444,8 @@ async function liveLeg(fail: (leg: string, line: string) => void, evidence: stri fail("live", "GOAL_LIVE=1 needs AI_GATEWAY_API_KEY"); return; } + // A container's FSDEV_DEFAULT_MODEL / FSDEV_INTENT_* would swap the answers' model under the resolver. + stripIntentOverrides(); // `goals/` cannot resolve the gateway package; kitchen-sink's node_modules can. const ksRequire = createRequire(new URL("../../../apps/kitchen-sink/package.json", import.meta.url)); const { createGateway } = (await import(ksRequire.resolve("@ai-sdk/gateway"))) as { @@ -364,8 +466,7 @@ async function liveLeg(fail: (leg: string, line: string) => void, evidence: stri const io = reader(app, OWNER); const posts: Array<{ body: string; want?: string }> = [ { body: "Hi, my laptop won't join the office wifi since this morning.", want: devices }, - // The follow-up names no subject; best fit routes it on its own words (goal.md, "Retired legs"). - { body: "It sees it. It fails right after the password." }, + { body: "It sees it. It fails right after the password.", want: devices }, { body: "My phone stopped charging, and it's a brand new cable.", want: devices }, { body: "Where can I buy it?" }, { body: "Different thing: I was charged twice for my subscription this month.", want: accounts },