diff --git a/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/fixtures/input.json b/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/fixtures/input.json index 61ff399f54..6991fe8efc 100644 --- a/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/fixtures/input.json +++ b/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/fixtures/input.json @@ -6,6 +6,36 @@ "ask": "Ask researcher for our launch date, and help me outline the post.", "second": "Make section 2 about pricing.", "fix": "Have coder fix the typo in the README's first line.", + "remember": "By the way, the draft of the launch post is filed under the codename \"{code}\". Nothing to do about it, just say OK.", + "recall": "What codename did I give the launch post, and what is our launch date? Answer from what you already know, without asking anyone.", + "codename": { + "colours": ["amber", "cobalt", "crimson", "jade", "ochre", "violet", "silver", "teal", "umber", "saffron"], + "animals": ["heron", "otter", "lynx", "badger", "falcon", "marmot", "gecko", "walrus", "ibis", "tapir"], + "numbers": [214, 377, 452, 519, 608, 731, 846, 903, 128, 265] + }, + "fillers": [ + "In one sentence, why does bread go stale?", + "Give me one tip for keeping houseplants alive.", + "In one sentence, how does a bicycle stay upright?", + "Name a good stretch for a stiff neck, briefly.", + "In one sentence, what makes a sourdough starter work?", + "Suggest a name for a small black cat.", + "In one sentence, why is the sea salty?", + "What is a quick way to chill a drink?", + "In one sentence, how do noise-cancelling headphones work?", + "Give me a one-line idea for a rainy Sunday.", + "In one sentence, why do onions make you cry?", + "What is one way to remember people's names?", + "In one sentence, how does a heat pump heat a house?", + "Suggest a simple breakfast with eggs, in one line.", + "In one sentence, why do leaves change colour in autumn?", + "Give me one tip for writing a clear email.", + "In one sentence, what does a thermostat do?", + "Name one board game for two players, briefly.", + "In one sentence, why does ice float?", + "What is a good habit for better sleep, in one line?" + ], + "pastWindow": 60, "project": { "id": "relay", "title": "Relay" }, "typos": [ { "readme": "# Relay — a tiny relay servce", "wrong": "servce", "right": "service" }, diff --git a/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/goal.md b/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/goal.md index 3bc219833a..87f4190d24 100644 --- a/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/goal.md +++ b/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/goal.md @@ -1,15 +1,15 @@ # teammates-lab › it a worker talks to another while you talk -**Issue:** FIX-2154 (leg b of the FIX-2152 epic's goal); step 5 and the held-out date amended by FIX-2309 -**Outcome:** In the Teammates lab, a list-of-workers app on the Grok Bot pattern, you talk to lead, lead messages researcher, and researcher's reply reaches lead while you are still talking to it; lead uses it. A coding job lead hands coder runs through Claude Code in the user's one project, which lead puts its conversation in without asking. -**Input:** held out. The launch date is drawn at random each run and written into the project's README, which researcher reads and lead can't, so lead can only state it by hearing it from researcher. The README's typo is one of several (`fixtures/input.json`). The lines you send are in the fixture. +**Issue:** FIX-2154 (leg b of the FIX-2152 epic's goal); step 5 and the held-out date amended by FIX-2309; step 4 with FIX-1613 +**Outcome:** In the Teammates lab, a list-of-workers app on the Grok Bot pattern, you talk to lead, lead messages researcher, and researcher's reply reaches lead while you are still talking to it; lead uses it. Long after both have left lead's history window, lead still knows them, and the screen shows where its word-for-word reading begins. A coding job lead hands coder runs through Claude Code in the user's one project, which lead puts its conversation in without asking. +**Input:** held out. The launch date is drawn at random each run and written into the project's README, which researcher reads and lead can't, so lead can only state it by hearing it from researcher. Step 4's codename (a colour, an animal and a number) is drawn per run, and its 60 lines are shuffled. The README's typo is one of several (`fixtures/input.json`). The lines you send are in the fixture. **Signal:** - **1:** lead answers your first line in its conversation. - **2:** a card *To researcher* in lead's conversation, a line *From lead* in researcher's, the exchange under *with researcher* in lead's Talk panel, and a delivered row lead → researcher in the message log. - **3:** your second line goes in before researcher's reply has landed. The reply shows in lead's conversation under researcher's name, after lead's answer to you, carrying the date; the log holds it as a delivered reply; a line of lead's after it states the date. -- **4:** not run. It needs the history window's mark (G1, FIX-1613); the step waits on it. +- **4:** you tell lead a codename, then send 60 more lines, past the 50-turn history window, each answered before the next. Then you ask for the codename and the launch date. lead's answer holds all three words of the codename and the date, and lead sent researcher nothing new for it (no new card, no new log row). The conversation shows a divider labelled *Earlier turns summarized*, and both the codename line and researcher's reply are drawn above it. - **5:** you ask for the fix. With one project, lead puts its conversation in `relay` and files the task there: no card asks for a project, and the header shows `relay`. The run ends *done in relay*, its steps show Claude Code, and the result lands in lead's conversation from coder. On disk: the checkout's HEAD and working tree are untouched, and exactly one worktree of it, under the lab's runs folder, carries a commit that changes README.md alone, fixing its first line and nothing else. -**Anti-game:** Graded on what the browser shows, the message log read through the public client, and git. Never on the lab's adapter or the server's own state. Lead can't state the launch date without the reply, because it is drawn per run. The disk check reads git, so a run that only says it fixed the typo fails. The step-3 order check reads entry order as drawn, so a reply drawn before lead's answer fails. Each graded list is floored with `tell.over`; an empty log or no worktree counts only where a control produces it, and says so. +**Anti-game:** Graded on what the browser shows, the message log read through the public client, and git. Never on the lab's adapter or the server's own state. Lead can't state the launch date without the reply, because it is drawn per run. The disk check reads git, so a run that only says it fixed the typo fails. The step-3 order check reads entry order as drawn, so a reply drawn before lead's answer fails. Step 4 counts lead's messages to researcher around the question, so asking researcher again for the date fails, and it reads which side of the divider each fact sits on, so a fact still read word for word doesn't count. Each graded list is floored with `tell.over`; an empty log or no worktree counts only where a control produces it, and says so. **Model:** real — openai/gpt-5.4-mini (lead, researcher, writer); coder's run on Claude Code **Run:** `pnpm tsx goals/teammates-lab/a-worker-talks-to-another-while-you-talk/run.mts` (needs `AI_GATEWAY_API_KEY`, and Claude Code signed in or `ANTHROPIC_API_KEY`) **Controls:** each on the same command, failing the steps named and no other: @@ -17,6 +17,10 @@ - `GOAL_CONTROL=no-lead-send`: lead's `sendMessage` is dropped from its tools (a scratch patch to `message-tool.ts`). Steps 2 and 3 FAIL. Step 5 may fail too: with no way to message, lead files its research ask as a task for coder, and after the pick it filed nothing again (2 of 2 runs). - `GOAL_CONTROL=no-reply-tool`: researcher's `sendMessage` is dropped. Step 3 FAILS. - `GOAL_CONTROL=project-not-kept`: `setWorkProject` answers as kept but stores nothing (a scratch patch to `work-project.ts`), so lead's task has no project and the README never changes. Step 5 FAILS. Step 3 may fail too: researcher can't keep the project it reads the date from. +- `GOAL_CONTROL=no-summary`: the history summary's capability and fold are dropped from the lab's host (a scratch patch to `host.mts`); memory stays. Step 4 FAILS: the divider can't say *summarized*. +- `GOAL_CONTROL=no-memory`: the summary and memory are both dropped from the host. Step 4 FAILS: nothing holds what left the window. + +Step 4 is skipped under `no-model`, `no-lead-send`, `no-reply-tool` and `project-not-kept`: its 60 lines test nothing about the step each control reddens. Steps 2 and 3 have no isolating red state for lead's side: without lead's message there is nothing to reply to, so `no-lead-send` reddens both. Under `no-model` the screen doesn't show your own lines either until a reload: a turn that fails shows nothing live. @@ -31,6 +35,16 @@ Steps 2 and 3 have no isolating red state for lead's side: without lead's messag | 2026-10-28 | 42fdeb2+ | openai/no-such-model-goal-control | FAIL (expected) | `no-model`: [1] no answer (and CAN'T TELL on 1, nothing drawn), [2], [3], [5] all red | | 2026-10-28 | 920da34+ | openai/gpt-5.4-mini, Claude Code | PASS | After FIX-2289's Markdown entries; the goal reads entries as shown (innerText). Typo "servce"; date 5 May; one worktree, README.md only | | 2026-10-28 | 7bd40d0+ | openai/gpt-5.4-mini, Claude Code | PASS | Streamed sends. Lead told never to pass `wait`, researcher that only `sendMessage` reaches the asker (earlier runs failed [3] when lead called and when researcher answered in place); step 2 waits for the Talk panel to read the log. Typo "servce"; date 21 April | +| 2026-10-28 | aace8e4+ | openai/gpt-5.4-mini, Claude Code | FAIL | Step 4 only: the fold stored the summarizer's `summary` and dropped its `keyPoints`, where the date went (fixed in FIX-1613, 6693940); a codename filed as a standing rule (now a fail of its own) | +| 2026-10-28 | bb004bc+ | openai/gpt-5.4-mini, Claude Code | FAIL | Step 4 only: the second fold rewrote the summary from the newer turns and lost the launch facts; the lab's summarizer objectives now say its summary replaces the one so far | +| 2026-10-28 | 068ae50 | openai/gpt-5.4-mini, Claude Code | PASS | Steps 1 to 5. Step 4: "cobalt walrus 608" and 17 September after 60 lines; 4 folds, both facts in the stored summary; divider "Earlier turns summarized" below both | +| 2026-10-28 | 068ae50 | openai/gpt-5.4-mini | FAIL (expected) | `no-summary`: [4] "I don't have that information", divider "Earlier turns out of view". A first run also reddened [3]: researcher answered in its own conversation, not with `sendMessage` | +| 2026-10-28 | 068ae50 | openai/gpt-5.4-mini | FAIL (expected) | `no-memory`: [4] neither fact, divider "Earlier turns out of view"; [3] red the same way as above | +| 2026-10-28 | a2caae9+ | openai/gpt-5.4-mini, Claude Code | PASS | Steps 1 to 5 after FIX-2289's Markdown entries, entries read as laid out. Step 4: "umber badger 214" and 4 December | +| 2026-10-28 | 14fa6b0 | openai/gpt-5.4-mini, Claude Code | FAIL | Step 4 only, a read too early: lead answered "crimson walrus 846" and 22 May, but with FIX-2289's streamed answers "is answering" cleared before the answer's last words were drawn, and the goal read "…and the". The goal now waits for the drawn answer to settle | +| 2026-10-28 | 00e8f07 | openai/gpt-5.4-mini, Claude Code | PASS | Steps 1 to 5 with streamed answers, the answer read once drawn in full. Step 4: "cobalt gecko 377" and 22 February; divider "Earlier turns summarized" below both | | 2026-10-28 | 600e888+ | openai/gpt-5.4-mini, Claude Code | PASS | Under the lab's own Grok Bot-like theme (FIX-2308). Typo "mesages"; date 10 February | | 2026-10-28 | 8141924+ | openai/gpt-5.4-mini, Claude Code | PASS | Real tools, no hard-coded facts: researcher reads the date from the README. Lead filed into relay with no question. Typo "servce"; date 11 August; one worktree, README.md only | | 2026-10-29 | 8141924+ | openai/gpt-5.4-mini | FAIL (expected) | `project-not-kept`: [5] task ended needing a project, a card asked, header shows none, 0 worktrees; [3] also (mayAlso): no reply from researcher | +| 2026-10-29 | 5e1a3f9 | openai/gpt-5.4-mini, Claude Code | FAIL | After merging FIX-2309 (real workers). [3] researcher never replied within 240s, so [4] had no reply to place above the divider; codename and date were still answered. Model, not the merge | +| 2026-10-29 | 5e1a3f9 | openai/gpt-5.4-mini, Claude Code | PASS | Steps 1 to 5 on the real-tools lab. Step 4: "teal badger 519" and 19 May; divider "Earlier turns summarized" below both. Step 5 filed in relay with no question. Typo "mesages" | diff --git a/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/run.mts b/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/run.mts index c6f5728b26..9ca806666e 100644 --- a/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/run.mts +++ b/goals/teammates-lab/a-worker-talks-to-another-while-you-talk/run.mts @@ -17,7 +17,10 @@ * shows in lead's conversation under researcher's name after lead's answer * to you, a delivered reply row is in the log, and a line of lead's after * it carries the held-out launch date. - * 4 not run: the history window's mark is gap G1 (FIX-1613). + * 4 you give lead a codename, then 60 more lines, past the 50-turn window; + * lead answers the codename and the launch date without asking anyone, + * and the divider, labelled summarized, sits after both the codename + * line and researcher's reply. * 5 you ask for a fix; with one project, lead puts its conversation in * `relay` and files the task there with no question, the run goes through * Claude Code, a worktree of the checkout gets one commit that changes the @@ -25,11 +28,11 @@ * and the result lands in lead's conversation from coder. * * Inputs are held out: the launch date is drawn at random and written into the - * project's README, where researcher reads it, and the README's typo is one of - * several. + * project's README, where researcher reads it, the codename is drawn at random, and the + * README's typo is one of several. * * Run: pnpm tsx goals/teammates-lab/a-worker-talks-to-another-while-you-talk/run.mts - * Controls: GOAL_CONTROL=no-model | no-lead-send | no-reply-tool | project-not-kept (see goal.md) + * Controls: GOAL_CONTROL=no-model | no-lead-send | no-reply-tool | project-not-kept | no-summary | no-memory (see goal.md) */ import { spawn, execFileSync, type ChildProcess } from "node:child_process"; import { writeFileSync, mkdirSync } from "node:fs"; @@ -51,6 +54,7 @@ import { runGoal, stopProcessGroup, type ModulePatch, + type Tell, } from "../../lib/index.mts"; import { launchChromium } from "../../lib/playwright.mts"; @@ -62,6 +66,13 @@ interface Fixture { ask: string; second: string; fix: string; + /** Step 4's first line; `{code}` is the codename drawn for the run. */ + remember: string; + recall: string; + codename: { colours: string[]; animals: string[]; numbers: number[] }; + fillers: string[]; + /** How many lines step 4 sends between the codename and the question: past the 50-turn window. */ + pastWindow: number; project: { id: string; title: string }; typos: { readme: string; wrong: string; right: string }[]; rest: string; @@ -82,19 +93,26 @@ const dropSendFor = (worker: string): ModulePatch => ({ to: `if (typeof self !== "string" || self === ${JSON.stringify(worker)}) return [];`, }); +/** A scratch patch to the lab's host, where lead's flow is put together. */ +const hostPatch = (from: string): ModulePatch => ({ module: "labs/teammates/host.mts", from, to: "" }); +/** The history summary out of lead's flow: its capability and its fold. */ +const NO_SUMMARY = [hostPatch(String.raw`,\s*earlier\.capability`), hostPatch(String.raw`beforeAnswer: earlier\.fold,`)]; + /** * Each control: what it changes, the steps it must redden, and any step it may * redden besides, with the reason in a comment. Any other red step is a leak. + * `skip` is step 4 under a control about another step: its 60 turns test + * nothing there. */ -const CONTROLS: Record = { +const CONTROLS: Record = { // No model answers: lead's conversation holds only your own lines. - "no-model": { model: "openai/no-such-model-goal-control", steps: ["1", "2", "3", "5"] }, + "no-model": { model: "openai/no-such-model-goal-control", steps: ["1", "2", "3", "5"], skip: ["4"] }, // lead can't message: no card, no line in researcher's conversation, no row, and so no reply. // Step 5 may go red too: with no way to message, lead files its research ask as a task instead, // and after the pick it has filed none again (seen in 2 of 2 runs). That is the model, not a leak. - "no-lead-send": { patches: [dropSendFor(fixture.lead)], steps: ["2", "3"], mayAlso: ["5"] }, + "no-lead-send": { patches: [dropSendFor(fixture.lead)], steps: ["2", "3"], mayAlso: ["5"], skip: ["4"] }, // researcher hears lead but can't answer it. - "no-reply-tool": { patches: [dropSendFor(fixture.researcher)], steps: ["3"] }, + "no-reply-tool": { patches: [dropSendFor(fixture.researcher)], steps: ["3"], skip: ["4"] }, // A conversation's project answers as kept but is never stored, so lead's task has no project. // Step 3 may go red too: researcher puts its own conversation in the project to read the date, // and that is not stored either. @@ -107,6 +125,21 @@ const CONTROLS: Record { // As shown: block breaks between the label and a Markdown body, and the label in its uppercase style. text: (el as HTMLElement).innerText, status: el.querySelector("[data-task-status]")?.textContent ?? undefined, + inWindow: el.closest("[data-in-window]")?.getAttribute("data-in-window") ?? undefined, })), ); } @@ -234,8 +270,103 @@ async function open(page: Page, worker: string): Promise { } const userAt = (entries: Drawn[], text: string) => entries.findIndex((e) => e.kind === "user" && e.text.includes(text)); +const lastUserAt = (entries: Drawn[], text: string) => entries.map((e) => e.kind === "user" && e.text.includes(text)).lastIndexOf(true); const short = (id: string) => id.replace(/^teammates\./, ""); +/** Say `text` to lead and wait until lead has answered it and finished the turn; false when it didn't in time. */ +async function turnWith(page: Page, lead: string, text: string): Promise { + const said = (es: Drawn[]) => es.filter((e) => e.kind === "user" && e.text.includes(text)).length; + const before = said(await entriesOf(page)); + await tell(page, lead, text); + // The line is drawn once its turn starts; only an answer after this copy of it counts. + const answered = await readUntil( + page, + (es) => said(es) > before && es.slice(lastUserAt(es, text) + 1).some((e) => e.kind === "assistant" && e.text.trim() !== ""), + fixture.answerWithinMs, + ); + if (!answered.ok) return false; + // Memory records the turn after the answer, in the same request: the next line waits for it. + const busy = conversation(page).getByRole("status").filter({ hasText: "is answering" }); + for (const until = Date.now() + fixture.answerWithinMs; Date.now() < until && (await busy.count()) > 0; ) await sleep(250); + if ((await busy.count()) > 0) return false; + // "is answering" clears when the request ends, which can be before the streamed answer's last words are drawn. + const shown = async () => (await entriesOf(page)).map((e) => e.text).join("\n"); + for (let was = await shown(), until = Date.now() + 10_000; Date.now() < until; ) { + await sleep(1_000); + const now = await shown(); + if (now === was) break; + was = now; + } + return true; +} + +/** + * Step 4: a codename, then more lines than the window holds, then the codename + * and the launch date asked back. Both left the window long before the + * question, so lead answers them from what it kept, and the divider says where + * its word-for-word reading begins. + */ +async function stepFour( + page: Page, + log: () => Promise<{ from: string; to: string }[]>, + date: ReturnType, + fail: (step: string, line: string) => void, + evidence: string[], + tell_: Tell, +): Promise { + const lead = short(fixture.lead); + const researcher = short(fixture.researcher); + const code = { colour: pick(fixture.codename.colours), animal: pick(fixture.codename.animals), number: pick(fixture.codename.numbers) }; + const codename = `${code.colour} ${code.animal} ${code.number}`; + const remember = fixture.remember.replace("{code}", codename); + const fillers = [...fixture.fillers].sort(() => Math.random() - 0.5); + console.error(`goal: step 4 codename "${codename}", then ${fixture.pastWindow} lines`); + for (const [i, line] of [remember, ...Array.from({ length: fixture.pastWindow }, (_, n) => fillers[n % fillers.length]!)].entries()) { + if (!(await turnWith(page, lead, line))) { + fail("4", `lead didn't finish answering line ${i + 1} of step 4 within ${fixture.answerWithinMs / 1000}s: "${line}"`); + return; + } + } + + // A standing rule is in lead's view on every turn, so a codename kept as one never left it. + const afterRemember = await entriesOf(page); + if (afterRemember.slice(userAt(afterRemember, remember) + 1).some((e) => /Rule added/.test(e.text) && e.text.includes(code.animal))) { + fail("4", "lead kept the codename as a standing rule, so it never left lead's view and the step tests nothing"); + } + + // Asked back, from what lead kept: a fresh message to researcher would answer the date without it. + const asked = (await log()).filter((r) => r.from === fixture.lead && r.to === fixture.researcher).length; + const card = conversation(page).getByRole("button", { name: new RegExp(`^To ${researcher}: `) }); + const cards = await card.count(); + if (!(await turnWith(page, lead, fixture.recall))) { + fail("4", `lead didn't answer the question within ${fixture.answerWithinMs / 1000}s`); + return; + } + const drawn = tell_.over("4 · entries drawn in lead's conversation", await entriesOf(page)); + const answer = drawn.slice(lastUserAt(drawn, fixture.recall) + 1).find((e) => e.kind === "assistant")?.text ?? ""; + const missing = [code.colour, code.animal, String(code.number)].filter((word) => !answer.toLowerCase().includes(word)); + if (missing.length > 0) fail("4", `lead's answer lacks ${missing.join(", ")} of the codename "${codename}": "${answer.slice(0, 300)}"`); + if (!date.isIn(answer)) fail("4", `lead's answer doesn't state the launch date ${date.text}: "${answer.slice(0, 300)}"`); + if ((await card.count()) > cards || (await log()).filter((r) => r.from === fixture.lead && r.to === fixture.researcher).length > asked) { + fail("4", "lead messaged researcher again to answer the question"); + } + + // The divider: labelled summarized, with the codename line and researcher's reply above it. + const divider = conversation(page).getByRole("separator"); + const shown = await divider.first().waitFor({ timeout: 15_000 }).then(() => true, () => false); + const label = shown ? await divider.first().getAttribute("aria-label") : null; + if (!shown) fail("4", "no divider in lead's conversation"); + else if (label !== "Earlier turns summarized") fail("4", `the divider reads "${label}", not "Earlier turns summarized"`); + const marked = await entriesOf(page); + const codeLine = marked[userAt(marked, remember)]; + const reply = marked.find((e) => e.kind === "received" && e.label === `From ${researcher}`); + if (codeLine?.inWindow !== "false") fail("4", "the codename line is not above the divider, so lead may have read it word for word"); + if (reply?.inWindow !== "false") fail("4", "researcher's reply is not above the divider, so lead may have read the date word for word"); + if (missing.length === 0 && date.isIn(answer) && label === "Earlier turns summarized") { + evidence.push(`4: after ${fixture.pastWindow} more lines lead answered "${codename}" and ${date.text}; divider "${label}" below both`); + } +} + // --------------------------------------------------------------------------- await runGoal(async (failures, tell_) => { @@ -373,8 +504,9 @@ await runGoal(async (failures, tell_) => { if (used !== undefined) evidence.push(`3: reply landed after lead's answer; lead's next line says ${date.text}`); } - // ---- 4 · the history window ------------------------------------------------ - evidence.push("4: not run, waits on the history window's mark (G1, FIX-1613)"); + // ---- 4 · past the history window -------------------------------------------- + if (control?.skip?.includes("4")) evidence.push(`4: not run under GOAL_CONTROL=${CONTROL}`); + else await stepFour(page, log, date, fail, evidence, tell_); // ---- 5 · lead hands coder a fix, in the one project ---------------------- const before = diskOf(checkout); diff --git a/labs/teammates/README.md b/labs/teammates/README.md index da337441b7..584c933686 100644 --- a/labs/teammates/README.md +++ b/labs/teammates/README.md @@ -85,6 +85,11 @@ and help me outline the post."* lead answers and messages researcher, which read README. Before researcher replies, send lead a second line: the reply shows as pending, then lands after lead's answer, and lead uses it in its next line. +A conversation outlasts its history window (50 turns). Before a turn leaves it, the turn is folded +into a running summary that lead reads with the turns it still has word for word. The divider sits +where those begin, labelled *summarized*, so a fact you gave lead a hundred lines ago is above it and +lead still knows it. + Then ask lead *"Have coder fix the typo in the README's first line."* lead puts its conversation in your project and files the task there, coder's run shows its steps, and the result lands in lead's conversation. @@ -107,11 +112,10 @@ names the package call it makes. | Hiring, changing and firing your own teammates | the workforce client's `hire`, `edit` and `fire`, on the roster flow | | Task cards, the refusal for want of a project | the conversation board's rows, and `projectWorkspace()`'s `no-project` refusal | | coder's runs | the task row's `run` link, read as the run session's items | +| The divider and its *summarized* label | the last request's `historyStart`, and the session's `historySummary` resource | Not wired yet: -- **The divider and its *summarized* label** wait on the history window's mark (G1, FIX-1613). Until - then the screen draws no divider. - **Which harness coder runs on.** The roster doesn't carry a worker's settings, so the screen names the harness from the run's own steps once it starts (FIX-2281). - **Editing a worker from the files.** They are read-only on the roster; how a user changes one (a @@ -129,8 +133,8 @@ and on a flow over 200 lines; `--wired` also fails on any fixture. | Kind | Files | Lines | |---|---:|---:| -| Configuration | 18 | 887 | -| React UI | 26 | 2615 | +| Configuration | 16 | 636 | +| React UI | 24 | 2269 | | Small flows | 1 | 152 | -| Checks (not app code) | 8 | 1043 | +| Checks (not app code) | 9 | 1097 | diff --git a/labs/teammates/host.mts b/labs/teammates/host.mts index aee7522d37..0da9fcb5d1 100644 --- a/labs/teammates/host.mts +++ b/labs/teammates/host.mts @@ -7,6 +7,9 @@ * - **lead, researcher and writer** run on the built-in `agent` flow, so each * has `sendMessage` and one conversation where messages land. Memory is * composed in through `uses` (read) and `afterAnswer` (record each turn). + * A history summary keeps what was said before the history window: its + * capability goes in `uses` and its fold runs as `beforeAnswer`, so a turn + * is summarized before it leaves the window the worker reads word for word. * The flow's tool catalog holds Workforce's project tools, web search and * fetch, and `uses` adds reading a project's source; each worker's * `tools:` line picks its own from them. @@ -33,7 +36,7 @@ import { CLAUDE_CODE_PASS, claudeCodeAgent } from "@flow-state-dev/claude-code/s import { createFlowState, runAction } from "@flow-state-dev/engine"; import { system } from "@flow-state-dev/memory"; import { sqliteStores } from "@flow-state-dev/store-sqlite"; -import { defineFlow } from "@flow-state-dev/core"; +import { defineFlow, utility } from "@flow-state-dev/core"; import { DEFAULT_ORG_ID } from "@flow-state-dev/core/types"; import { fetch, search } from "@flow-state-dev/tools"; import { @@ -125,6 +128,19 @@ export async function openTeammates(options: TeammatesOptions) { ); const mem = system({ model: options.model, working: { capacity: 7 }, episodic: true, semantic: true }); + const earlier = utility.historySummary({ + summarize: utility.summarizer({ + name: "earlier-turns", + model: options.model, + granularity: "detailed", + // Named, or a small model summarizes the newer turns alone and drops the summary so far. + objectives: [ + "The input's `summary` is the summary so far and `turns` are newer turns: write one summary of both, never of the turns alone.", + "Your summary replaces the summary so far: everything it says must still be in yours, then add what the newer turns add.", + "Keep every name, date, code and number the user or another worker stated, word for word. Small talk can shrink; those never go.", + ], + }), + }); // One host for coder's runs and the workers' reads, so a read sees the clone a run works from. const workspace = localWorkspaceHost({ root: options.runsRoot, @@ -147,7 +163,8 @@ export async function openTeammates(options: TeammatesOptions) { fetch: fetch(), }, // The project collections the source reads go through. - uses: [mem.capability.presets({ recall: false, connect: false, semantic: true, episodic: true }), projectWorkspaceCapability], + uses: [mem.capability.presets({ recall: false, connect: false, semantic: true, episodic: true }), projectWorkspaceCapability, earlier.capability], + beforeAnswer: earlier.fold, afterAnswer: mem.captureFromItems, }), [CODER_FLOW]: coderFlow({ diff --git a/labs/teammates/src/adapter.ts b/labs/teammates/src/adapter.ts index b4a82b6beb..2be4077d68 100644 --- a/labs/teammates/src/adapter.ts +++ b/labs/teammates/src/adapter.ts @@ -12,7 +12,7 @@ * | `answerApproval` | the request's `answer`: approve or reject a `user_approval` stop | * | `watchTalk` | the user's message log (`workforceMessages`), as `sendMessage` writes it | * | `watchActivity` | `listSessionRequests` on the worker's conversation | - * | `watchWindow` | the mark the history window leaves: gap G1 (FIX-1613), not yet in the package | + * | `watchWindow` | the last answering request's `historyStart`, and the session's `historySummary` | * | `listProjects` | the user's projects, read through their roster session | * | `watchProjects` | the same read, again every few seconds, so a project lead made shows | * | `createProject` | the projects flow's `createProject`, the block lead's tool runs | @@ -236,6 +236,46 @@ const parseArgs = (item: Item): Record => { } }; +/** The session resource the lab's history summary keeps; its public face carries the mark. */ +const HISTORY_SUMMARY = "historySummary"; + +/** What the screen reads of where the worker's last turn began reading word for word. */ +interface WindowRead { + /** The first turn the window held, when earlier turns had left it. */ + historyStart?: string; + /** The newest turn folded into the summary, once one was. */ + summarizedThrough?: string; + /** Whether the summary covers every turn before its mark; one with a hole can't be called that. */ + summaryComplete?: boolean; +} + +/** + * The divider, from the items: before the first entry the worker read word + * for word. That is the window's first turn, or the first turn after the + * summary's mark when the summary reaches past it: the capability starts the + * reads there, so no turn is read twice. Turns go in the order they started, + * as history reads them: overlapping turns interleave their items, so the + * summarized turn's last item can sit after a newer turn's first. + */ +export function windowMarkOf(items: readonly Item[], entries: readonly ConversationEntry[], read: WindowRead): WindowMark | undefined { + const firstOf = (requestId: string) => items.findIndex((i) => i.requestId === requestId); + const turns = [...new Set(items.map((i) => i.requestId))]; + /** The first item of the turn that started after `requestId`; past the end when none did. */ + const turnAfter = (requestId: string) => { + const at = turns.indexOf(requestId); + if (at < 0) return -1; + const next = turns[at + 1]; + return next === undefined ? items.length : firstOf(next); + }; + const start = read.historyStart === undefined ? -1 : firstOf(read.historyStart); + const folded = read.summarizedThrough === undefined ? -1 : turnAfter(read.summarizedThrough); + const from = Math.max(start, folded); + if (from < 0) return undefined; + const drawn = new Set(entries.map((e) => e.id)); + const first = items.slice(from).find((i) => drawn.has(i.id)); + return first === undefined ? undefined : { firstVerbatimEntryId: first.id, summarized: read.summaryComplete !== false && folded >= 0 && folded >= start }; +} + /** Read `read` now and every `everyMs` until stopped; a failed read keeps the last value. */ function poll(read: () => Promise, onValue: (value: T) => void, everyMs = POLL_MS): Unsubscribe { let live = true; @@ -518,10 +558,45 @@ export function createLabAdapter(client: Client = connect()): LabAdapter { return activityOf(await client.sessions.requests(known.sessionId, { limit: 20 })); }, onChange); }, - watchWindow(_sessionId, onChange) { - // G1 (FIX-1613): the package leaves no mark yet, so no divider is drawn. - onChange(undefined); - return () => {}; + watchWindow(sessionId, onChange) { + let items: Item[] = []; + let read: WindowRead = {}; + let last = ""; + const emit = () => { + const mark = windowMarkOf(items, toEntries(items, workerIds), read); + const seen = JSON.stringify(mark ?? null); + if (seen === last) return; + last = seen; + onChange(mark); + }; + const stopFeed = watchFeed(sessionId, (next) => next, (next) => { + items = next; + emit(); + }); + const stopPoll = poll( + async (): Promise => { + // The latest turn of the worker's own, by when it started: its window is the one the worker read last. + const answering = (await client.sessions.requests(sessionId, { limit: 20 })).filter((r) => !QUIET_ACTIONS.has(r.actionName)); + const latest = answering.reduce((a, r) => (a === undefined || (r.startedAtMs ?? 0) > (a.startedAtMs ?? 0) ? r : a), undefined); + const summary = (await client.sessions.state(sessionId, { includeItems: false })).resources?.session?.[HISTORY_SUMMARY] as + | { clientData?: { throughRequestId?: string | null; complete?: boolean } } + | undefined; + const through = summary?.clientData?.throughRequestId; + return { + ...(latest?.historyStart == null ? {} : { historyStart: latest.historyStart.requestId }), + // The worker reads after the mark whether or not the summary has a hole; a hole only changes the label. + ...(through == null ? {} : { summarizedThrough: through, summaryComplete: summary?.clientData?.complete === true }), + }; + }, + (next) => { + read = next; + emit(); + }, + ); + return () => { + stopFeed(); + stopPoll(); + }; }, listProjects, watchProjects(onChange) { diff --git a/labs/teammates/test/window-mark.test.ts b/labs/teammates/test/window-mark.test.ts new file mode 100644 index 0000000000..6b577672ec --- /dev/null +++ b/labs/teammates/test/window-mark.test.ts @@ -0,0 +1,51 @@ +/** + * The divider's place: before the first entry the worker read word for word. + * A worker with a history summary reads from the turn after the summary's + * mark, so a divider at the window's first turn would show turns as read that + * were only summarized; one labelled summarized with no summary would promise + * the worker remembers turns it dropped. + */ +import { describe, expect, it } from "vitest"; +import { toEntries, windowMarkOf } from "../src/adapter"; + +const user = (requestId: string) => ({ id: `${requestId}_you`, requestId, type: "message", role: "user", content: [{ text: `you in ${requestId}` }] }); +const answer = (requestId: string) => ({ id: `${requestId}_lead`, requestId, type: "message", role: "assistant", content: [{ text: `lead in ${requestId}` }] }); +/** An item the conversation doesn't draw, first in its turn. */ +const hidden = (requestId: string) => ({ id: `${requestId}_hidden`, requestId, type: "reasoning", content: [] }); + +const items = ["r1", "r2", "r3", "r4"].flatMap((r) => [hidden(r), user(r), answer(r)]); +const entries = toEntries(items, new Set()); + +describe("the history window's divider", () => { + it("draws none while every turn is in the window", () => { + expect(windowMarkOf(items, entries, {})).toBeUndefined(); + }); + + it("sits before the window's first turn, out of view, when there is no summary", () => { + expect(windowMarkOf(items, entries, { historyStart: "r3" })).toEqual({ firstVerbatimEntryId: "r3_you", summarized: false }); + }); + + it("sits after the summary's mark, labelled summarized, when the summary reaches into the window", () => { + // The window still holds r2, but r2 is in the summary, so the worker reads from r3. + expect(windowMarkOf(items, entries, { historyStart: "r2", summarizedThrough: "r2" })).toEqual({ firstVerbatimEntryId: "r3_you", summarized: true }); + }); + + it("is summarized before any turn has left the window, since the summary folds early", () => { + expect(windowMarkOf(items, entries, { summarizedThrough: "r1" })).toEqual({ firstVerbatimEntryId: "r2_you", summarized: true }); + }); + + it("sits before the first turn that started after the mark when turns overlap", () => { + // r1 answers after r2 has begun: r2 is read word for word, so the divider can't sit after r1's answer. + const overlapped = [user("r1"), user("r2"), answer("r2"), answer("r1"), user("r3"), answer("r3")]; + expect(windowMarkOf(overlapped, toEntries(overlapped, new Set()), { summarizedThrough: "r1" })).toEqual({ firstVerbatimEntryId: "r2_you", summarized: true }); + }); + + it("still sits after the mark, but not labelled summarized, when the summary has a hole", () => { + // The worker reads only after r2 whatever the summary covers; a hole means it can't claim to remember r1 and r2. + expect(windowMarkOf(items, entries, { historyStart: "r2", summarizedThrough: "r2", summaryComplete: false })).toEqual({ firstVerbatimEntryId: "r3_you", summarized: false }); + }); + + it("calls turns out of view when the window starts past the summary's mark", () => { + expect(windowMarkOf(items, entries, { historyStart: "r4", summarizedThrough: "r2" })).toEqual({ firstVerbatimEntryId: "r4_you", summarized: false }); + }); +}); diff --git a/packages/core/src/types/block.ts b/packages/core/src/types/block.ts index 0810834b11..fd447f8f07 100644 --- a/packages/core/src/types/block.ts +++ b/packages/core/src/types/block.ts @@ -736,10 +736,12 @@ export interface BlockContext< * {@link _emitComponentAwaited}, settling once the item has been added and * completed and rejecting rather than swallowing. Absent in a hand-built * context; a caller that depends on it treats absence as a failure. + * `role` defaults to `assistant`; `user` records input the flow received + * from someone else, such as another worker's message. */ _emitMessageAwaited?( text: string, - options?: { itemVisibility?: ItemVisibility; agentName?: string; transient?: boolean } + options?: { itemVisibility?: ItemVisibility; agentName?: string; transient?: boolean; role?: "user" | "assistant" } ): Promise; /** diff --git a/packages/engine/src/context/createExecutionContext.ts b/packages/engine/src/context/createExecutionContext.ts index 26462f8776..994b0dbcb8 100644 --- a/packages/engine/src/context/createExecutionContext.ts +++ b/packages/engine/src/context/createExecutionContext.ts @@ -415,7 +415,10 @@ function createEmitComponentAwaited( */ function createEmitMessageAwaited( emCtx: EmissionContext -): (text: string, options?: { itemVisibility?: ItemVisibility; agentName?: string; transient?: boolean }) => Promise { +): ( + text: string, + options?: { itemVisibility?: ItemVisibility; agentName?: string; transient?: boolean; role?: "user" | "assistant" } +) => Promise { return async function emitMessageAwaited(text, options): Promise { const itemIndex = emCtx.nextItemIndex(); const item: MessageItem = { @@ -431,7 +434,7 @@ function createEmitMessageAwaited( taskId: emCtx.taskId, itemVisibility: options?.itemVisibility ?? emCtx.itemVisibility, agentName: options?.agentName ?? emCtx.agentName, - role: "assistant", + role: options?.role ?? "assistant", content: [{ type: "output_text", text }] }; await emCtx.response.emitItemAdded(item); diff --git a/packages/workforce/README.md b/packages/workforce/README.md index ddc2883df1..e43aeba276 100644 --- a/packages/workforce/README.md +++ b/packages/workforce/README.md @@ -2330,7 +2330,8 @@ Send to talk; ask for a reply when the worker wants an answer but can go on; cal below, and so does a call. - **Receiving.** A message arrives on the recipient flow's internal `onMessageReceived` entry (`MESSAGE_RECEIVED_ENTRY`). The session it lands in gets two items: a `From : ` - line under the sender's name that the model reads in history and the client never receives, and + `user` line under the sender's name that the model reads in history as someone else's words, and + the client never receives, and a client-only component item `workforce:received` (`RECEIVED_MESSAGE_COMPONENT`) with `data` `{ id, from, text, inReplyTo?, budget, call, replyRequested? }`, for an app to draw. Then the flow's turn, if it gave one, runs with a `MessageReceived` (`{ id, from, text, call, inReplyTo?, budget, replyRequested? }`; `call` is `true` when the diff --git a/packages/workforce/src/messages/message-received-entry.ts b/packages/workforce/src/messages/message-received-entry.ts index 10758f8cb6..e7459d13fe 100644 --- a/packages/workforce/src/messages/message-received-entry.ts +++ b/packages/workforce/src/messages/message-received-entry.ts @@ -204,7 +204,9 @@ export function messageReceivedEntry(options: { // The two items, in one place, after the take: the line the model // reads, and the item an app draws. const emit = storedEmitters(ctx as never); - await emit.message(`From ${row.from}: ${row.text}`, { agentName: row.from, itemVisibility: { client: false, history: true } }); + // A `user` line: the sender's words, which the recipient's model must + // not read as something it already said. + await emit.message(`From ${row.from}: ${row.text}`, { role: "user", agentName: row.from, itemVisibility: { client: false, history: true } }); await emit.component( RECEIVED_MESSAGE_COMPONENT, { diff --git a/packages/workforce/test/worker-messages.test.ts b/packages/workforce/test/worker-messages.test.ts index b662bd6c6d..7ff57a68ad 100644 --- a/packages/workforce/test/worker-messages.test.ts +++ b/packages/workforce/test/worker-messages.test.ts @@ -282,6 +282,33 @@ describe("an answer", () => { await h.dispose(); } }); + + it("hands the recipient's model a heard message as another's words, never as its own", async () => { + // Stored as an assistant line, the model reads the sender's request as + // something it already said, and can end its turn with nothing to add. + const agentAnswer = mockGenerator({ + script: [ + { when: (input) => textOf(input).includes("It is Bluebird."), then: { text: "The codename is Bluebird." } }, + ...callOnce("ask the lead", "sendMessage", { to: "lead", text: "What is the codename?" }), + fallback + ] + }); + const h = boot({ + judgment: mockGenerator({ script: [...answerOnce("otto", "What is the codename?", "It is Bluebird."), fallback] }), + agentAnswer + }); + try { + const otto = await agentSession(h, "alice", "otto"); + await h.post("alice", otto, "respond", { message: "Please ask the lead" }, "agent"); + await h.settled(); + + const woken = agentAnswer.calls.find((call) => textOf(call.input).includes("It is Bluebird."))!; + const heard = (woken.input as Array<{ role: string; content: unknown }>).filter((m) => JSON.stringify(m.content).includes("From lead: It is Bluebird.")); + expect(heard.map((m) => m.role)).toEqual(["user"]); + } finally { + await h.dispose(); + } + }); }); /** The board's workers, a coordinator that files for `otto`, and two more agent workers. */