From a2c8a4f1eae49248428464f748697bc04427bc3c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:00:42 +0000 Subject: [PATCH 01/24] spec(FIX-2398): what each task and turn cost Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- specs/issues/FIX-2398/DECISIONS.md | 89 ++++++++++++++++ specs/issues/FIX-2398/SPEC.md | 156 +++++++++++++++++++++++++++++ 2 files changed, 245 insertions(+) create mode 100644 specs/issues/FIX-2398/DECISIONS.md create mode 100644 specs/issues/FIX-2398/SPEC.md diff --git a/specs/issues/FIX-2398/DECISIONS.md b/specs/issues/FIX-2398/DECISIONS.md new file mode 100644 index 0000000000..f2c09d776d --- /dev/null +++ b/specs/issues/FIX-2398/DECISIONS.md @@ -0,0 +1,89 @@ +# FIX-2398 · Decisions + +[Spec](SPEC.md) · **Decisions** + +FIX-2288 put a task's spend on its row and every harness reports into it. These three say what +"usage for every harness" means when two vendors send no reading mid-run, where a Kitchen Sink +turn's spend comes from, and what a cost is. + +## The tree + +```mermaid +flowchart TD + I["FIX-2398"] --> D1["D1 · no onUsage for Codex and Cursor"] + D1 -.->|"rejected"| X1["an onUsage option on both · nothing on the wire to carry"] + I --> D2["D2 · a turn's spend is its request's ledger at turn end"] + D2 -.->|"rejected"| X2["a per-turn feed the harness calls · a second report path"] + D2 -.->|"rejected"| X3["sum the turn's items · not what the vendor billed"] + I --> D3["D3 · a cost is the vendor's charge or nothing"] + D3 -.->|"rejected"| X4["a price-table estimate · forbidden by FIX-2145 D2"] +``` + +Solid edges are what you're signing. Dashed edges lost, and the label says why. + + +## D1 · Codex and Cursor get no `onUsage`; Teammates' gap is a reader and a dropped `onActivity` + +| | | +|---|---| +| **Instead of** | An `onUsage` option on `codexAgent` and `cursorAgent`, as the issue asks | +| **Because** | `onUsage` carries a subscription window a vendor announces mid-run. Claude Code sends `rate_limit_event`; Codex's wire has `thread.*`, `turn.*`, `item.*` and `error` only, and Cursor's has `usage` (tokens) only (`packages/codex/src/types.ts`, `packages/cursor/src/types.ts`). Their tokens and cost already reach every task row through `reportHarnessSpend`, in all three apps. Codex's windows come from `readCodexUsage` as a `harnessUsageAction` reader. What Teammates actually drops is Cursor's `onActivity` (its host passes `cwd, resume, onSession, onSteerable` only) and any reader | +| **Locks in** | An app that wants Codex's windows registers a reader; nothing arrives by itself. If Codex or Cursor later stream a window event, `onUsage` is added in that harness package then, one hook per real event | + +**What would change my mind:** a Codex or Cursor wire event carrying a limit reading. Then the +option is cheap and belongs in that package. + + +## D2 · A turn's spend is its request's ledger at turn end, folded by `taskSpendView` and kept on the turn record + +| | | +|---|---| +| **Instead of** | A feed callback on `HarnessSessionFeeds` the harness calls with its usage; or a sum over the turn's items | +| **Because** | Every harness already reports its run into the request's ledger, from its `finally`, however the run ends. The session flow's request `onFinished` runs after the harness block returns and before the ending hands the ledger on, so `readRequestSpend(ctx)` there is the whole turn: a failed or stopped turn included. Folding it with `taskSpendView` gives a turn the same view a task has, so one component draws both (tenet 5: one fold). A feed callback would be a second report path beside `reportHarnessSpend`, kept in step by every harness | +| **Locks in** | A turn's count is final, not live: it appears when the turn ends. The record's `spend` has the view's shape, so a change to the view changes both readers at once | + +**What would change my mind:** a page that must show a turn's tokens rising while it runs. Then +the turn writes its live part on a timer, as a task's request does (FIX-2288 D2). + + +## D3 · A cost is the vendor's billed charge, summed, or `null` + +| | | +|---|---| +| **Instead of** | `$0.00` for a subscription run, or a price-table estimate for a run with tokens and no charge | +| **Because** | FIX-2145 D2 and the epic forbid estimates. A subscription run is not billed per run, so a dollar figure for it would be invented. `costUsd` is the sum of the sources that reported one; `null` when none did, and the line then shows tokens only | +| **Locks in** | A mixed task (an API-key retry after a subscription run) shows the API-key part's charge, with tokens covering both. Apps never compute a cost | + +## Decided, not asked + +- **The turn's duration is `endedAt − at`.** `at` is already the turn's start; `endedAt` is + written in the same turn-end write as `spend`. A task's elapsed time is FIX-2394's, on FIX-2168's + row fields; the component takes `durationMs` so both feed it. +- **The registry component takes a structural summary**, the view's fields plus optional + `attempts` and `durationMs`, and no row type: a turn and a task call it the same way. A + harness label is the app's (`harnessLabel`), so the registry names no harness. +- **The session flow carries the usage query** (`harnessUsage`, readers from `usageReaders`, + default none). The collection is not client-readable, so a write with no query on the flow + is invisible. +- **Kitchen Sink installs the byte copy in PR 1**: its drift check requires every registry file. +- **A turn-end write that fails is a note, never a failed turn**, like the pull-request check. + +## Considered and dropped + +| Alternative | Why it lost | +|---|---| +| Keep each lab's own spend line | Four copies of one line, which ER-3 says go to the package | +| Put `costUsd` on FIX-2379's row view only | The row view carries the spend view (ER-4); the cost belongs in the fold it carries | +| Write the turn spend from a request-end hook | It runs after the request is over, so a page that refreshes on the turn's end can read before it lands; `onFinished` runs after the harness reported and before the request ends | + +## Analysis + +2026-10-31, read from `main` 675f614. **Right approach.** Value: the four apps' "what did it cost" +line. Reuse: every Adds row composes an existing fold or writer ([Builds on / Adds](SPEC.md#whats-already-here-and-the-debt)). +Scope: D1 narrows the issue's Codex/Cursor ask to what their wires carry (recorded). Debt: net +down after PR 5. + +## How it got here + +- **Drafted (Oct 31)** from the issue and epic FIX-2369's seams, with the Codex and Cursor wire + types read to settle D1. diff --git a/specs/issues/FIX-2398/SPEC.md b/specs/issues/FIX-2398/SPEC.md new file mode 100644 index 0000000000..2f81ba0771 --- /dev/null +++ b/specs/issues/FIX-2398/SPEC.md @@ -0,0 +1,156 @@ +# FIX-2398 · What each task and turn cost + +**Spec** · [Decisions](DECISIONS.md) + +## Five people, before and after + +| Someone who… | Today | After | +|---|---|---| +| **picks a harness for a coding task in Teammates** | Sees the harness and model the last run used, and nothing about what it cost. Choosing by cost is a guess | Sees tokens, the vendor's charge when there is one, and attempts on the card and in the popup, and a total for the Background list | +| **runs a Codex or Cursor task in any app** | Tokens reach the task row already, but no app but Claude Code-style draws them | The same line every harness gets: Codex and Cursor tokens and cost through the same view as Claude Code's | +| **works a Kitchen Sink harness session** | No turn records what it spent, and a Claude Code window warning heard mid-run is dropped | Each turn records its tokens, cost and duration; the window reading lands in the person's usage record, readable through the session flow's usage query | +| **opens a task in Shift Manager** | The inspector's *Harness · tokens · cost* section is a "not built yet" gap | The section shows the task's spend line from the row | +| **runs a subscription-billed harness** | n/a | Sees tokens and *no dollar figure*, never an estimate | + +The row already carries a task's spend (FIX-2288) and every harness already reports it. What is +missing is one way to draw it, the cost in the view, and the same record for a Kitchen Sink turn, +which has no task row. + +## The goal, and how we'll know it's met + +**Every app that shows a coding task or a harness turn shows what it spent, in the same words: +model, tokens, the vendor's charge when it reported one, and for a turn how long it took.** + +| Is it the right goal? | | +|---|---| +| **The real need** | Pick a harness by what it costs, and see a run's spend wherever the run shows ([FIX-2398](https://linear.app/fixpoint-labs/issue/FIX-2398), epic [FIX-2369](../../epics/FIX-2369/SPEC.md)'s *see what the work is doing*) | +| **Smaller, and rejected** | "Teammates draws the row's tokens." Hittable in one lab file, and leaves Kitchen Sink recording nothing and three more copies of the same line | +| **Bigger, and not this issue's** | A task's elapsed time and stall signals: FIX-2394, on the `startedAt`/`endedAt` row fields FIX-2168 adds. This ships the slot they fill. Subscription windows for Codex and Cursor: neither vendor sends one during a run ([D1](DECISIONS.md#d1)) | +| **Not done if** | The view has a cost but the line never shows it · a turn's count is read off a scripted harness only · a subscription run shows `$0.00` · a turn that failed or was stopped records nothing | + +```mermaid +flowchart LR + I["a real harness turn · harnessSessionFlow · api-key sign-in"] --> T["the turn's record in session state"] + T -->|"tokens > 0 and the vendor's cost, model named"| P["PASS · goal met"] + I --> U["a stopped turn"] + U -->|"its tokens recorded too"| P + C["control · no-turn-spend"] -.-> T + T -.->|"under the control"| F["must FAIL · turn spend is null"] +``` + +The check reads the turn record the page reads, through the flow's own state, never the +harness's handle. + +| How we verify | | +|---|---| +| **Goal check** | `goals/harness-manager/a-turn-records-what-it-spent/` · real Claude Code on an API key (so a charge is reported) · run by the implementer at completion where the host can run Claude Code, else handed to `fsd-qa` · verdict in the implementation PR | +| **Signal** | **t**: after a completed turn, its record's `spend` names the model, `inputTokens + outputTokens > 0`, `costUsd > 0`, and `endedAt ≥ at`. **s**: a stopped turn's record carries `spend` too | +| **Input** | A one-line prompt with a fresh token to echo. Any other prompt must pass | +| **Anti-game** | Read the session state the page reads. Don't assert on `HarnessRunSpend` or a mocked ledger: both pass while the turn record stays empty | +| **Control that must fail** | `GOAL_CONTROL=no-turn-spend` (the flow skips its turn-end write). **t** and **s** FAIL on `spend` being `null`. Today's `main` fails the same way | + +## What changes + +```mermaid +flowchart LR + H["claude-code · codex · cursor
reportHarnessSpend (unchanged)"] --> L["the request's spend ledger"] + L -->|"task runs (FIX-2288)"| R["task row spend"] + L -->|"turn end, new"| TR["turn record spend + endedAt"] + R --> V["taskSpendView · + costUsd"] + TR --> V + V --> S["registry Spend · one line"] + S --> A["Teammates · Shift Manager · CC lab · Kitchen Sink"] + CC["Claude Code onUsage"] -->|"session feeds, new"| HU["harness-usage record · harnessUsage action"] +``` + +One fold (`taskSpendView`) for a task and a turn, one component for every app. A turn's spend is +the request's own ledger at its end ([D2](DECISIONS.md#d2)). + +**A host, after:** + +```diff + harnessSessionFlow({ + kind: "claude-code-session", + workspace, ++ // The session flow's `harnessUsage` query reads what `onUsage` records. Optional. ++ usageReaders: { "claude-code": () => readClaudeUsage({ env }) }, + agent: (feeds) => claudeCodeAgent({ ...feeds, permissionMode: "bypassPermissions" }), + }); +``` + +```diff +- // a lab's own copy ++ // the registry's +``` + +**Docs:** harness-manager README (the turn record's `spend` and `endedAt`, the `onUsage` feed, +`usageReaders`), orchestration README (`costUsd` on the spend view), ui README (the `spend` +item). No new docs-site page: the harness session and task-board pages gain one line each. + +## The PR plan + +| PR | What | Base | +|---|---|---| +| 1 · package slice | `taskSpendView.costUsd`; the session flow's turn spend, `onUsage` feed and `harnessUsage` action; the registry `spend` item and Kitchen Sink's byte copy | FIX-2387's implementation branch (the turn-end region), else `main` | +| 2 · Kitchen Sink | `harness-session.tsx` draws the latest turn's line | After 2410 → 2386 → 2384 (wave 2) | +| 3 · Shift Manager | the inspector's section and Tasks' COST column | After PR 1 and FIX-2418 | +| 4 · Teammates, stack 3 | card, popup and Background total; Cursor gets `onActivity` | After 2380 → 2382 (wave 4) | +| 5 · Claude Code-style | its `Spend.tsx` becomes the registry copy | After PR 1 | + +## What stays as it is + +- How a harness reports: `reportHarnessSpend` and the request's ledger, untouched. Codex and + Cursor already report tokens and cost through it. +- The task row's `spend` field and its writes (FIX-2288). Only the fold gains a field. +- `recordHarnessUsage`'s rules. The session flow is one more writer through it, as the manager + and the session door already are. +- FIX-2145 D2: a cost is only ever the vendor's billed charge. + +## What's already here, and the debt + +| Already here | What this does with it | +|---|---| +| `taskSpendView`, the fold of a row's spend (FIX-2288) | **Extends**: `costUsd`, the sum of reported charges | +| `requestSpendSources`, `readRequestSpend` (FIX-2145, FIX-2288) | **Builds on**: the turn's spend is the request's pending spend at its end | +| `recordHarnessUsage`, `harnessUsageCollection`, `harnessUsageAction` | **Reuses**: the session feeds call the one writer; the session flow carries the one query | +| Claude Code-style `Spend.tsx` | **Replaces** (PR 5): lifted into the registry, taking a view, not a lab row | +| The turn record (`harnessTurnRecordSchema`) | **Extends**: `spend`, `endedAt` | +| Anything else that draws a task's tokens | **None** but CC lab's: searched `tokenCount`, `spendLine`, `inputTokens` across `labs/`, `apps/`, `packages/*/src` | +| FIX-2288 D3, model and harness from the run's report | **Kept** | +| Epic ER-4, consume the owner's artifact | **Kept**: the row view is FIX-2379's; this adds to the spend view it carries, not a second view | + +| Adds | Existing piece considered | Why it doesn't fit / how composed | +|---|---|---| +| `TaskSpendView.costUsd` | `TaskSpendSource.costUsd` | Composed: summed from the sources the view already folds | +| Turn `spend` and `endedAt` | `taskSpendView`; the task row | Composed: the turn stores `taskSpendView` of its own request's sources. A turn has no task row | +| `HarnessSessionFeeds.onUsage`, `usageReaders` | The manager's and the door's `onUsage` | Composed: the same one-line call into `recordHarnessUsage` | +| Registry `spend` item | CC lab's `Spend.tsx` | Lifted, so four apps stop copying it | + +| The debt | | +|---|---| +| **Removes** | (PR 5) Claude Code-style's own `Spend.tsx` and `tokenCount` | +| **Adds** | None: every addition reuses an existing fold or writer | +| **Leaves** | Two spend shapes and two cost rules for one run (FIX-2330). The manager run row's own token count (FIX-2298) | + +**Net: down** once PR 5 lands: one copy goes, nothing second is added. + +## Sign off + +**[The goal](#the-goal-and-how-well-know-its-met), at that size:** every app's task and turn shows +model, tokens, the vendor's charge and (for a turn) its duration, through one component. If wrong: +we ship one more lab copy, or hold this for task timing FIX-2394 already owns. + +**[The debt](#whats-already-here-and-the-debt), net down:** a lab copy goes; no second fold. If +wrong: a fifth spend line someone keeps in step. + +1. **[D1](DECISIONS.md#d1) · Codex and Cursor get no `onUsage`.** Their tokens already reach the + row; neither vendor sends a window reading mid-run. If wrong: a feed with nothing to carry, or + Teammates still missing Codex's windows, which a reader supplies. +2. **[D2](DECISIONS.md#d2) · A turn's spend is its request's ledger at turn end, folded by + `taskSpendView`.** If wrong: a second fold, or a harness callback beside `reportHarnessSpend`. +3. **[D3](DECISIONS.md#d3) · A cost is the vendor's charge or nothing.** If wrong: a subscription + run reads `$0.00`, or an estimate nobody billed. + +**Open: none.** Number 1 is the one to weigh. + +Feature · `orchestration` + `harness-manager` + `ui` registry, then four app PRs · small package slice · epic FIX-2369 From 9754783681950a275fdf60c6144725f81a8d466f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:00:42 +0000 Subject: [PATCH 02/24] spec(FIX-2398): what each task and turn cost Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- specs/issues/FIX-2398/DECISIONS.md | 89 ++++++++++++++++ specs/issues/FIX-2398/SPEC.md | 156 +++++++++++++++++++++++++++++ 2 files changed, 245 insertions(+) create mode 100644 specs/issues/FIX-2398/DECISIONS.md create mode 100644 specs/issues/FIX-2398/SPEC.md diff --git a/specs/issues/FIX-2398/DECISIONS.md b/specs/issues/FIX-2398/DECISIONS.md new file mode 100644 index 0000000000..f2c09d776d --- /dev/null +++ b/specs/issues/FIX-2398/DECISIONS.md @@ -0,0 +1,89 @@ +# FIX-2398 · Decisions + +[Spec](SPEC.md) · **Decisions** + +FIX-2288 put a task's spend on its row and every harness reports into it. These three say what +"usage for every harness" means when two vendors send no reading mid-run, where a Kitchen Sink +turn's spend comes from, and what a cost is. + +## The tree + +```mermaid +flowchart TD + I["FIX-2398"] --> D1["D1 · no onUsage for Codex and Cursor"] + D1 -.->|"rejected"| X1["an onUsage option on both · nothing on the wire to carry"] + I --> D2["D2 · a turn's spend is its request's ledger at turn end"] + D2 -.->|"rejected"| X2["a per-turn feed the harness calls · a second report path"] + D2 -.->|"rejected"| X3["sum the turn's items · not what the vendor billed"] + I --> D3["D3 · a cost is the vendor's charge or nothing"] + D3 -.->|"rejected"| X4["a price-table estimate · forbidden by FIX-2145 D2"] +``` + +Solid edges are what you're signing. Dashed edges lost, and the label says why. + + +## D1 · Codex and Cursor get no `onUsage`; Teammates' gap is a reader and a dropped `onActivity` + +| | | +|---|---| +| **Instead of** | An `onUsage` option on `codexAgent` and `cursorAgent`, as the issue asks | +| **Because** | `onUsage` carries a subscription window a vendor announces mid-run. Claude Code sends `rate_limit_event`; Codex's wire has `thread.*`, `turn.*`, `item.*` and `error` only, and Cursor's has `usage` (tokens) only (`packages/codex/src/types.ts`, `packages/cursor/src/types.ts`). Their tokens and cost already reach every task row through `reportHarnessSpend`, in all three apps. Codex's windows come from `readCodexUsage` as a `harnessUsageAction` reader. What Teammates actually drops is Cursor's `onActivity` (its host passes `cwd, resume, onSession, onSteerable` only) and any reader | +| **Locks in** | An app that wants Codex's windows registers a reader; nothing arrives by itself. If Codex or Cursor later stream a window event, `onUsage` is added in that harness package then, one hook per real event | + +**What would change my mind:** a Codex or Cursor wire event carrying a limit reading. Then the +option is cheap and belongs in that package. + + +## D2 · A turn's spend is its request's ledger at turn end, folded by `taskSpendView` and kept on the turn record + +| | | +|---|---| +| **Instead of** | A feed callback on `HarnessSessionFeeds` the harness calls with its usage; or a sum over the turn's items | +| **Because** | Every harness already reports its run into the request's ledger, from its `finally`, however the run ends. The session flow's request `onFinished` runs after the harness block returns and before the ending hands the ledger on, so `readRequestSpend(ctx)` there is the whole turn: a failed or stopped turn included. Folding it with `taskSpendView` gives a turn the same view a task has, so one component draws both (tenet 5: one fold). A feed callback would be a second report path beside `reportHarnessSpend`, kept in step by every harness | +| **Locks in** | A turn's count is final, not live: it appears when the turn ends. The record's `spend` has the view's shape, so a change to the view changes both readers at once | + +**What would change my mind:** a page that must show a turn's tokens rising while it runs. Then +the turn writes its live part on a timer, as a task's request does (FIX-2288 D2). + + +## D3 · A cost is the vendor's billed charge, summed, or `null` + +| | | +|---|---| +| **Instead of** | `$0.00` for a subscription run, or a price-table estimate for a run with tokens and no charge | +| **Because** | FIX-2145 D2 and the epic forbid estimates. A subscription run is not billed per run, so a dollar figure for it would be invented. `costUsd` is the sum of the sources that reported one; `null` when none did, and the line then shows tokens only | +| **Locks in** | A mixed task (an API-key retry after a subscription run) shows the API-key part's charge, with tokens covering both. Apps never compute a cost | + +## Decided, not asked + +- **The turn's duration is `endedAt − at`.** `at` is already the turn's start; `endedAt` is + written in the same turn-end write as `spend`. A task's elapsed time is FIX-2394's, on FIX-2168's + row fields; the component takes `durationMs` so both feed it. +- **The registry component takes a structural summary**, the view's fields plus optional + `attempts` and `durationMs`, and no row type: a turn and a task call it the same way. A + harness label is the app's (`harnessLabel`), so the registry names no harness. +- **The session flow carries the usage query** (`harnessUsage`, readers from `usageReaders`, + default none). The collection is not client-readable, so a write with no query on the flow + is invisible. +- **Kitchen Sink installs the byte copy in PR 1**: its drift check requires every registry file. +- **A turn-end write that fails is a note, never a failed turn**, like the pull-request check. + +## Considered and dropped + +| Alternative | Why it lost | +|---|---| +| Keep each lab's own spend line | Four copies of one line, which ER-3 says go to the package | +| Put `costUsd` on FIX-2379's row view only | The row view carries the spend view (ER-4); the cost belongs in the fold it carries | +| Write the turn spend from a request-end hook | It runs after the request is over, so a page that refreshes on the turn's end can read before it lands; `onFinished` runs after the harness reported and before the request ends | + +## Analysis + +2026-10-31, read from `main` 675f614. **Right approach.** Value: the four apps' "what did it cost" +line. Reuse: every Adds row composes an existing fold or writer ([Builds on / Adds](SPEC.md#whats-already-here-and-the-debt)). +Scope: D1 narrows the issue's Codex/Cursor ask to what their wires carry (recorded). Debt: net +down after PR 5. + +## How it got here + +- **Drafted (Oct 31)** from the issue and epic FIX-2369's seams, with the Codex and Cursor wire + types read to settle D1. diff --git a/specs/issues/FIX-2398/SPEC.md b/specs/issues/FIX-2398/SPEC.md new file mode 100644 index 0000000000..2f81ba0771 --- /dev/null +++ b/specs/issues/FIX-2398/SPEC.md @@ -0,0 +1,156 @@ +# FIX-2398 · What each task and turn cost + +**Spec** · [Decisions](DECISIONS.md) + +## Five people, before and after + +| Someone who… | Today | After | +|---|---|---| +| **picks a harness for a coding task in Teammates** | Sees the harness and model the last run used, and nothing about what it cost. Choosing by cost is a guess | Sees tokens, the vendor's charge when there is one, and attempts on the card and in the popup, and a total for the Background list | +| **runs a Codex or Cursor task in any app** | Tokens reach the task row already, but no app but Claude Code-style draws them | The same line every harness gets: Codex and Cursor tokens and cost through the same view as Claude Code's | +| **works a Kitchen Sink harness session** | No turn records what it spent, and a Claude Code window warning heard mid-run is dropped | Each turn records its tokens, cost and duration; the window reading lands in the person's usage record, readable through the session flow's usage query | +| **opens a task in Shift Manager** | The inspector's *Harness · tokens · cost* section is a "not built yet" gap | The section shows the task's spend line from the row | +| **runs a subscription-billed harness** | n/a | Sees tokens and *no dollar figure*, never an estimate | + +The row already carries a task's spend (FIX-2288) and every harness already reports it. What is +missing is one way to draw it, the cost in the view, and the same record for a Kitchen Sink turn, +which has no task row. + +## The goal, and how we'll know it's met + +**Every app that shows a coding task or a harness turn shows what it spent, in the same words: +model, tokens, the vendor's charge when it reported one, and for a turn how long it took.** + +| Is it the right goal? | | +|---|---| +| **The real need** | Pick a harness by what it costs, and see a run's spend wherever the run shows ([FIX-2398](https://linear.app/fixpoint-labs/issue/FIX-2398), epic [FIX-2369](../../epics/FIX-2369/SPEC.md)'s *see what the work is doing*) | +| **Smaller, and rejected** | "Teammates draws the row's tokens." Hittable in one lab file, and leaves Kitchen Sink recording nothing and three more copies of the same line | +| **Bigger, and not this issue's** | A task's elapsed time and stall signals: FIX-2394, on the `startedAt`/`endedAt` row fields FIX-2168 adds. This ships the slot they fill. Subscription windows for Codex and Cursor: neither vendor sends one during a run ([D1](DECISIONS.md#d1)) | +| **Not done if** | The view has a cost but the line never shows it · a turn's count is read off a scripted harness only · a subscription run shows `$0.00` · a turn that failed or was stopped records nothing | + +```mermaid +flowchart LR + I["a real harness turn · harnessSessionFlow · api-key sign-in"] --> T["the turn's record in session state"] + T -->|"tokens > 0 and the vendor's cost, model named"| P["PASS · goal met"] + I --> U["a stopped turn"] + U -->|"its tokens recorded too"| P + C["control · no-turn-spend"] -.-> T + T -.->|"under the control"| F["must FAIL · turn spend is null"] +``` + +The check reads the turn record the page reads, through the flow's own state, never the +harness's handle. + +| How we verify | | +|---|---| +| **Goal check** | `goals/harness-manager/a-turn-records-what-it-spent/` · real Claude Code on an API key (so a charge is reported) · run by the implementer at completion where the host can run Claude Code, else handed to `fsd-qa` · verdict in the implementation PR | +| **Signal** | **t**: after a completed turn, its record's `spend` names the model, `inputTokens + outputTokens > 0`, `costUsd > 0`, and `endedAt ≥ at`. **s**: a stopped turn's record carries `spend` too | +| **Input** | A one-line prompt with a fresh token to echo. Any other prompt must pass | +| **Anti-game** | Read the session state the page reads. Don't assert on `HarnessRunSpend` or a mocked ledger: both pass while the turn record stays empty | +| **Control that must fail** | `GOAL_CONTROL=no-turn-spend` (the flow skips its turn-end write). **t** and **s** FAIL on `spend` being `null`. Today's `main` fails the same way | + +## What changes + +```mermaid +flowchart LR + H["claude-code · codex · cursor
reportHarnessSpend (unchanged)"] --> L["the request's spend ledger"] + L -->|"task runs (FIX-2288)"| R["task row spend"] + L -->|"turn end, new"| TR["turn record spend + endedAt"] + R --> V["taskSpendView · + costUsd"] + TR --> V + V --> S["registry Spend · one line"] + S --> A["Teammates · Shift Manager · CC lab · Kitchen Sink"] + CC["Claude Code onUsage"] -->|"session feeds, new"| HU["harness-usage record · harnessUsage action"] +``` + +One fold (`taskSpendView`) for a task and a turn, one component for every app. A turn's spend is +the request's own ledger at its end ([D2](DECISIONS.md#d2)). + +**A host, after:** + +```diff + harnessSessionFlow({ + kind: "claude-code-session", + workspace, ++ // The session flow's `harnessUsage` query reads what `onUsage` records. Optional. ++ usageReaders: { "claude-code": () => readClaudeUsage({ env }) }, + agent: (feeds) => claudeCodeAgent({ ...feeds, permissionMode: "bypassPermissions" }), + }); +``` + +```diff +- // a lab's own copy ++ // the registry's +``` + +**Docs:** harness-manager README (the turn record's `spend` and `endedAt`, the `onUsage` feed, +`usageReaders`), orchestration README (`costUsd` on the spend view), ui README (the `spend` +item). No new docs-site page: the harness session and task-board pages gain one line each. + +## The PR plan + +| PR | What | Base | +|---|---|---| +| 1 · package slice | `taskSpendView.costUsd`; the session flow's turn spend, `onUsage` feed and `harnessUsage` action; the registry `spend` item and Kitchen Sink's byte copy | FIX-2387's implementation branch (the turn-end region), else `main` | +| 2 · Kitchen Sink | `harness-session.tsx` draws the latest turn's line | After 2410 → 2386 → 2384 (wave 2) | +| 3 · Shift Manager | the inspector's section and Tasks' COST column | After PR 1 and FIX-2418 | +| 4 · Teammates, stack 3 | card, popup and Background total; Cursor gets `onActivity` | After 2380 → 2382 (wave 4) | +| 5 · Claude Code-style | its `Spend.tsx` becomes the registry copy | After PR 1 | + +## What stays as it is + +- How a harness reports: `reportHarnessSpend` and the request's ledger, untouched. Codex and + Cursor already report tokens and cost through it. +- The task row's `spend` field and its writes (FIX-2288). Only the fold gains a field. +- `recordHarnessUsage`'s rules. The session flow is one more writer through it, as the manager + and the session door already are. +- FIX-2145 D2: a cost is only ever the vendor's billed charge. + +## What's already here, and the debt + +| Already here | What this does with it | +|---|---| +| `taskSpendView`, the fold of a row's spend (FIX-2288) | **Extends**: `costUsd`, the sum of reported charges | +| `requestSpendSources`, `readRequestSpend` (FIX-2145, FIX-2288) | **Builds on**: the turn's spend is the request's pending spend at its end | +| `recordHarnessUsage`, `harnessUsageCollection`, `harnessUsageAction` | **Reuses**: the session feeds call the one writer; the session flow carries the one query | +| Claude Code-style `Spend.tsx` | **Replaces** (PR 5): lifted into the registry, taking a view, not a lab row | +| The turn record (`harnessTurnRecordSchema`) | **Extends**: `spend`, `endedAt` | +| Anything else that draws a task's tokens | **None** but CC lab's: searched `tokenCount`, `spendLine`, `inputTokens` across `labs/`, `apps/`, `packages/*/src` | +| FIX-2288 D3, model and harness from the run's report | **Kept** | +| Epic ER-4, consume the owner's artifact | **Kept**: the row view is FIX-2379's; this adds to the spend view it carries, not a second view | + +| Adds | Existing piece considered | Why it doesn't fit / how composed | +|---|---|---| +| `TaskSpendView.costUsd` | `TaskSpendSource.costUsd` | Composed: summed from the sources the view already folds | +| Turn `spend` and `endedAt` | `taskSpendView`; the task row | Composed: the turn stores `taskSpendView` of its own request's sources. A turn has no task row | +| `HarnessSessionFeeds.onUsage`, `usageReaders` | The manager's and the door's `onUsage` | Composed: the same one-line call into `recordHarnessUsage` | +| Registry `spend` item | CC lab's `Spend.tsx` | Lifted, so four apps stop copying it | + +| The debt | | +|---|---| +| **Removes** | (PR 5) Claude Code-style's own `Spend.tsx` and `tokenCount` | +| **Adds** | None: every addition reuses an existing fold or writer | +| **Leaves** | Two spend shapes and two cost rules for one run (FIX-2330). The manager run row's own token count (FIX-2298) | + +**Net: down** once PR 5 lands: one copy goes, nothing second is added. + +## Sign off + +**[The goal](#the-goal-and-how-well-know-its-met), at that size:** every app's task and turn shows +model, tokens, the vendor's charge and (for a turn) its duration, through one component. If wrong: +we ship one more lab copy, or hold this for task timing FIX-2394 already owns. + +**[The debt](#whats-already-here-and-the-debt), net down:** a lab copy goes; no second fold. If +wrong: a fifth spend line someone keeps in step. + +1. **[D1](DECISIONS.md#d1) · Codex and Cursor get no `onUsage`.** Their tokens already reach the + row; neither vendor sends a window reading mid-run. If wrong: a feed with nothing to carry, or + Teammates still missing Codex's windows, which a reader supplies. +2. **[D2](DECISIONS.md#d2) · A turn's spend is its request's ledger at turn end, folded by + `taskSpendView`.** If wrong: a second fold, or a harness callback beside `reportHarnessSpend`. +3. **[D3](DECISIONS.md#d3) · A cost is the vendor's charge or nothing.** If wrong: a subscription + run reads `$0.00`, or an estimate nobody billed. + +**Open: none.** Number 1 is the one to weigh. + +Feature · `orchestration` + `harness-manager` + `ui` registry, then four app PRs · small package slice · epic FIX-2369 From 3383687d34c0d3c9ac2335d32cb10bed69365afb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:10:55 +0000 Subject: [PATCH 03/24] spec(FIX-2398): name the docs page that changes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- specs/issues/FIX-2398/SPEC.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/issues/FIX-2398/SPEC.md b/specs/issues/FIX-2398/SPEC.md index 2f81ba0771..e57e3d4626 100644 --- a/specs/issues/FIX-2398/SPEC.md +++ b/specs/issues/FIX-2398/SPEC.md @@ -85,7 +85,7 @@ the request's own ledger at its end ([D2](DECISIONS.md#d2)). **Docs:** harness-manager README (the turn record's `spend` and `endedAt`, the `onUsage` feed, `usageReaders`), orchestration README (`costUsd` on the spend view), ui README (the `spend` -item). No new docs-site page: the harness session and task-board pages gain one line each. +item), and the docs site's *Flow-aware components* page gains a *Spend* section. No new page. ## The PR plan From ddf60c86a5cac22f7e8a4bcb36afc87f94933979 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:14:28 +0000 Subject: [PATCH 04/24] feat(harness-manager, orchestration, ui): what each task and turn cost (FIX-2398) - taskSpendView gains costUsd: the sum of vendor-reported charges, null when none (a subscription run reports none; never an estimate). - harnessSessionFlow records each turn's spend (taskSpendView's shape) and endedAt at turn end, however the turn ended; hands harnesses an onUsage feed into the usage record; carries a harnessUsage action (usageReaders). - ui registry gains a spend item, lifted from Claude Code-style's Spend; Kitchen Sink installs its byte copy. - Goal check goals/harness-manager/a-turn-records-what-it-spent: PASS on real Claude Code; GOAL_CONTROL=no-turn-spend FAILs t and s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- .changeset/turn-and-task-cost.md | 6 + apps/docs/docs/ui/flow-aware-components.md | 17 +++ .../components/flow-state/spend.tsx | 92 +++++++++++++ apps/kitchen-sink/test/harness-turn.test.ts | 2 +- apps/kitchen-sink/test/registry-drift.test.ts | 4 +- .../a-turn-records-what-it-spent/goal.md | 27 ++++ .../a-turn-records-what-it-spent/run.mts | 109 +++++++++++++++ packages/harness-manager/README.md | 4 +- packages/harness-manager/src/session-flow.ts | 85 +++++++++++- .../harness-manager/test/turn-spend.spec.ts | 129 ++++++++++++++++++ packages/orchestration/README.md | 6 +- .../orchestration/src/tasks/schema/spend.ts | 7 + packages/orchestration/src/tasks/spend.ts | 6 +- .../test/collection/record-spend.test.ts | 19 ++- .../test/skills/list-tasks-spend.test.ts | 2 +- .../test/task-board/task-spend-run.test.ts | 4 +- packages/ui/README.md | 1 + packages/ui/registry.json | 14 ++ packages/ui/registry/components/spend.tsx | 92 +++++++++++++ packages/ui/test/render/spend.test.tsx | 68 +++++++++ 20 files changed, 679 insertions(+), 15 deletions(-) create mode 100644 .changeset/turn-and-task-cost.md create mode 100644 apps/kitchen-sink/components/flow-state/spend.tsx create mode 100644 goals/harness-manager/a-turn-records-what-it-spent/goal.md create mode 100644 goals/harness-manager/a-turn-records-what-it-spent/run.mts create mode 100644 packages/harness-manager/test/turn-spend.spec.ts create mode 100644 packages/ui/registry/components/spend.tsx create mode 100644 packages/ui/test/render/spend.test.tsx diff --git a/.changeset/turn-and-task-cost.md b/.changeset/turn-and-task-cost.md new file mode 100644 index 0000000000..1e719297a3 --- /dev/null +++ b/.changeset/turn-and-task-cost.md @@ -0,0 +1,6 @@ +--- +"@flow-state-dev/orchestration": minor +"@flow-state-dev/harness-manager": minor +--- + +`taskSpendView` gains `costUsd`: the sum of the charges vendors reported for the task, or `null` when none did (a subscription run reports none). `harnessSessionFlow` records what each turn spent on its turn record (`spend`, in `taskSpendView`'s shape, and `endedAt`), hands harnesses an `onUsage` feed that records subscription windows in the user's usage record, and carries a `harnessUsage` action that reads it, with optional `usageReaders`. diff --git a/apps/docs/docs/ui/flow-aware-components.md b/apps/docs/docs/ui/flow-aware-components.md index 7bf27fcc2e..bfae4d01e7 100644 --- a/apps/docs/docs/ui/flow-aware-components.md +++ b/apps/docs/docs/ui/flow-aware-components.md @@ -61,6 +61,23 @@ import { ModelBadge } from "@/components/flow-state/model-badge"; ``` +## Spend + +One line for what a coding run spent: the model, the tokens, and the vendor's charge when it reported one, like `claude-sonnet-4-5 · 12.4k tokens · $0.42`. Hand it the `spend` a task row carries (from `listTasks`), or the `spend` on a harness session's turn record; both have the same shape. A count nobody reported reads as "no tokens reported yet", never 0. A run on a subscription isn't billed per run, so it shows tokens and no dollar figure. + +```bash +fsdev ui add spend +``` + +```tsx +import { Spend } from "@/components/flow-state/spend"; + + + +``` + +`split` shows input and output apart. `attempts` appears from the second attempt on. `harnessLabel` puts the harness's name first, for a view that doesn't already show it. + ## AuditAnnotation A compact card for [Response Auditor](/docs/patterns/response-auditor) findings. It lists the surfaced results with severity indicators, per-analyzer scores, and the evidence each analyzer flagged. You don't render it by hand: the `responseAuditor` pattern emits an `audit-annotation` component item whenever a result surfaces, and `chatAssistantRenderers` maps that item to this card, so the audit shows up automatically on any chat surface wired with those renderers. diff --git a/apps/kitchen-sink/components/flow-state/spend.tsx b/apps/kitchen-sink/components/flow-state/spend.tsx new file mode 100644 index 0000000000..95d61af25b --- /dev/null +++ b/apps/kitchen-sink/components/flow-state/spend.tsx @@ -0,0 +1,92 @@ +"use client"; + +import { cn } from "@/lib/utils"; + +/* + * What a run spent, as one line: the model, the tokens, the vendor's charge + * when it reported one, and optionally the attempts and how long it took. The + * numbers are read from what the server folded (a task row's or a harness + * turn's `spend`), never summed here. A count nobody reported reads as such, + * never as 0; one still missing a source is marked partial. A run billed by + * subscription reports no charge and shows none: there is no dollar figure + * to show, and an estimate would be one nobody billed. + */ + +/** + * What the line reads: the fields of orchestration's `TaskSpendView`, which a + * task row and a harness session's turn both carry. `costUsd` absent reads as + * none reported. + */ +export interface SpendSummary { + harness: string | null; + model: string | null; + inputTokens: number | null; + outputTokens: number | null; + costUsd?: number | null; + unreported: boolean; + live: boolean; +} + +/** What else the line can say. */ +export interface SpendLineOptions { + /** Input and output apart ("10.1k in · 2.3k out") rather than one total. */ + split?: boolean; + /** The harness's name, first, for a view with no harness tag of its own. */ + harnessLabel?: string; + /** The task's attempts; shown from the second on. */ + attempts?: number; + /** How long it took, in ms. */ + durationMs?: number | null; +} + +/** 980, 12.4k, 1.2M. */ +export function tokenCount(n: number): string { + if (n < 1000) return String(n); + if (n < 1_000_000) return `${(n / 1000).toFixed(1)}k`; + return `${(n / 1_000_000).toFixed(1)}M`; +} + +/** $0.0042, $0.42, $12.40. */ +export function costAmount(usd: number): string { + return usd < 0.01 ? `$${usd.toFixed(4)}` : `$${usd.toFixed(2)}`; +} + +/** 42s, 4m 12s, 1h 3m. */ +export function durationText(ms: number): string { + const seconds = Math.max(0, Math.round(ms / 1000)); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ${seconds % 60}s`; + return `${Math.floor(minutes / 60)}h ${minutes % 60}m`; +} + +/** + * "claude-sonnet-4-5 · 12.4k tokens · $0.42 · 2 attempts · 4m 12s". Empty + * when there is no spend to show. + */ +export function spendLine(spend: SpendSummary | null | undefined, options: SpendLineOptions = {}): string { + if (spend == null) return ""; + const partial = spend.unreported ? " (partial)" : ""; + const side = (n: number | null) => (n === null ? "not reported" : tokenCount(n)); + const tokens = + spend.inputTokens === null && spend.outputTokens === null + ? "no tokens reported yet" + : options.split === true + ? `${side(spend.inputTokens)} in · ${side(spend.outputTokens)} out${partial}` + : `${tokenCount((spend.inputTokens ?? 0) + (spend.outputTokens ?? 0))} tokens${partial}`; + const cost = spend.costUsd == null ? null : costAmount(spend.costUsd); + const attempts = options.attempts !== undefined && options.attempts > 1 ? `${options.attempts} attempts` : null; + const duration = options.durationMs == null ? null : durationText(options.durationMs); + return [options.harnessLabel, spend.model, tokens, cost, attempts, duration].filter(Boolean).join(" · "); +} + +/** The spend line, or nothing when there is no spend to show. */ +export function Spend({ spend, className, ...options }: SpendLineOptions & { spend: SpendSummary | null | undefined; className?: string }) { + const line = spendLine(spend, options); + if (!line) return null; + return ( + + {line} + + ); +} diff --git a/apps/kitchen-sink/test/harness-turn.test.ts b/apps/kitchen-sink/test/harness-turn.test.ts index 43124c54cf..9dbce3b8eb 100644 --- a/apps/kitchen-sink/test/harness-turn.test.ts +++ b/apps/kitchen-sink/test/harness-turn.test.ts @@ -26,7 +26,7 @@ import { type PlanPanelInput, } from "@/lib/harness-turn"; -const blank: HarnessTurnRecord = { asked: null, confirmed: null, failure: null, stopped: null, error: null, at: 1, steered: null }; +const blank: HarnessTurnRecord = { asked: null, confirmed: null, failure: null, stopped: null, error: null, at: 1, steered: null, spend: null, endedAt: null }; describe("turnNotice", () => { it("says nothing while the turn runs", () => { diff --git a/apps/kitchen-sink/test/registry-drift.test.ts b/apps/kitchen-sink/test/registry-drift.test.ts index 86fe515e77..ae271a2162 100644 --- a/apps/kitchen-sink/test/registry-drift.test.ts +++ b/apps/kitchen-sink/test/registry-drift.test.ts @@ -33,11 +33,11 @@ describe("kitchen-sink registry drift (FIX-1477 V8)", () => { it("has every registry component byte-identical to its installed kitchen-sink copy", () => { const { compared, mismatches } = drift(); - // 37 of the registry's component-directory files (all non-`.stories.tsx` + // 38 of the registry's component-directory files (all non-`.stories.tsx` // files outside `generative/`) are installed here. If that count moves, // this test's scope moved with it and needs re-reading before trusting a // green result. - expect(compared.length).toBe(37); + expect(compared.length).toBe(38); expect(mismatches).toEqual([]); }); diff --git a/goals/harness-manager/a-turn-records-what-it-spent/goal.md b/goals/harness-manager/a-turn-records-what-it-spent/goal.md new file mode 100644 index 0000000000..3201031966 --- /dev/null +++ b/goals/harness-manager/a-turn-records-what-it-spent/goal.md @@ -0,0 +1,27 @@ +# harness-manager › a turn records what it spent + +**Issue:** FIX-2398 (epic FIX-2369) + +**Outcome:** Every app that shows a coding task or a harness turn shows what it spent, in the same words: model, tokens, the vendor's charge when it reported one, and for a turn how long it took. This check covers the turn half at the package: a harness session's turn record carries what the turn spent, however the turn ended. + +**Input:** a one-line prompt that asks Claude Code to echo a fresh random token, and a long count to stop mid-run. Any other prompt must pass. An in-memory store. + +**Signal:** real Claude Code, in process, signed in with an API key (`ANTHROPIC_API_KEY`, else `MY_ANTHROPIC_API_KEY`) so its vendor reports a charge. No tools. + +- **t**: after a completed turn, its record's `spend` names the model, `inputTokens + outputTokens > 0`, `costUsd > 0`, and `endedAt ≥ at`. +- **s**: a turn stopped after 8 s still has `spend`, naming `claude-code`. + +**Anti-game:** both legs read the turn record from session state, the record a page reads. Nothing reads the harness's handle or its spend report, which exist whether or not the record was written. + +**Model:** `claude-haiku-4-5` (override with `GOAL_CLAUDE_MODEL`). Any model Claude Code runs must pass. + +**Run:** `pnpm tsx goals/harness-manager/a-turn-records-what-it-spent/run.mts` + +**Controls:** on the same command. + +- `GOAL_CONTROL=no-turn-spend`: the flow's turn-end step is dropped, as a flow that never writes a turn's spend would. **t** and **s** FAIL on `spend` being null. + +## Verdict log +| Date | Commit | Model | Verdict | Notes | +|------|--------|-------|---------|-------| +| 2026-10-31 | implementation branch, on 45cefc5 | claude-haiku-4-5 (API key) | **PASS** | Linux container. **t**: 17,634 in · 73 out · $0.022405, 4.0 s. **s**: stopped, 17,491 in · 4 out, no charge reported for the stopped run. Control `no-turn-spend` FAILED **t** (no spend, no `endedAt`) and **s** (no spend) | diff --git a/goals/harness-manager/a-turn-records-what-it-spent/run.mts b/goals/harness-manager/a-turn-records-what-it-spent/run.mts new file mode 100644 index 0000000000..6f3299fddc --- /dev/null +++ b/goals/harness-manager/a-turn-records-what-it-spent/run.mts @@ -0,0 +1,109 @@ +/** + * Goal check — harness-manager › a turn records what it spent (FIX-2398). + * + * A real Claude Code turn runs through `harnessSessionFlow`, signed in with an + * API key so the vendor reports a charge. The check reads the turn's record + * from session state, the record a page reads, and never the harness's handle + * or its spend report: a report that never reaches the record is the failure + * this guards against. + * + * Legs: + * - **t**: a completed turn's `spend` names the model, has tokens and a + * charge above zero, and `endedAt` is at or after the turn's start. + * - **s**: a turn stopped mid-run still has `spend`, naming Claude Code. + * + * Control: `GOAL_CONTROL=no-turn-spend` drops the flow's turn-end step, as a + * flow that never writes a turn's spend would. **t** and **s** FAIL on + * `spend` being null. + * + * Run: pnpm tsx goals/harness-manager/a-turn-records-what-it-spent/run.mts + */ +import { randomBytes } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DEFAULT_ORG_ID, harnessEnv } from "@flow-state-dev/core"; +import { claudeCodeAgent } from "@flow-state-dev/claude-code/sdk"; +import { createInMemoryStores, runAction } from "@flow-state-dev/engine"; +import { harnessSessionFlow, type HarnessSessionState, type HarnessTurnRecord } from "@flow-state-dev/harness-manager"; +import { runGoal } from "../../lib/index.mts"; + +const control = process.env.GOAL_CONTROL ?? ""; +if ((process.env.ANTHROPIC_API_KEY ?? "") === "" && (process.env.MY_ANTHROPIC_API_KEY ?? "") !== "") { + process.env.ANTHROPIC_API_KEY = process.env.MY_ANTHROPIC_API_KEY; +} +const MODEL = process.env.GOAL_CLAUDE_MODEL ?? "claude-haiku-4-5"; +const STOP_AFTER_MS = 8_000; + +await runGoal(async (failures) => { + if ((process.env.ANTHROPIC_API_KEY ?? "") === "") { + return { failures: ["blocked: no ANTHROPIC_API_KEY (or MY_ANTHROPIC_API_KEY): the check needs an API-key sign-in so a charge is reported"], evidence: "" }; + } + const scratch = mkdtempSync(join(tmpdir(), "turn-spend-")); + const flow = harnessSessionFlow({ + kind: "turn-spend-goal", + workspace: { provision: async () => scratch, locate: () => scratch, branch: () => null }, + agent: (feeds) => + claudeCodeAgent({ + ...feeds, + // Only what the run needs: its sign-in and a shell's basics. No tools, so a turn is a reply. + env: harnessEnv({ pass: ["PATH", "HOME", "USER", "TMPDIR", "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "HTTPS_PROXY", "NODE_EXTRA_CA_CERTS"] }), + allowedTools: [], + disallowedTools: ["Bash", "Read", "Write", "Edit", "Grep", "Glob", "WebFetch", "WebSearch", "Task", "TodoWrite"], + settingSources: [], + detached: true, + } as never) as never, + }); + if (control === "no-turn-spend") (flow as { request?: unknown }).request = undefined; + + const stores = createInMemoryStores(); + const born = async (sessionId: string) => { + const now = Date.now(); + await stores.session.set( + sessionId, + { id: sessionId, flowKind: "turn-spend-goal", userId: "goal-user", orgId: DEFAULT_ORG_ID, metadata: undefined, latestRequestId: undefined, state: { model: MODEL }, version: 0, createdAt: now, updatedAt: now, journal: [] } as never, + "any" as never, + ); + }; + const turnOf = (sessionId: string, prompt: string, signal?: AbortSignal) => + runAction({ flow, actionName: "turn", input: { prompt }, userId: "goal-user", orgId: DEFAULT_ORG_ID, sessionId, stores, runtimeConfig: {}, ...(signal ? { signal } : {}) } as never); + const recordOf = async (sessionId: string): Promise => { + const state = (await stores.session.get(sessionId))?.state as HarnessSessionState | undefined; + return Object.values(state?.turns ?? {})[0]; + }; + + const evidence: string[] = []; + try { + // t — a completed turn. + const token = randomBytes(6).toString("hex"); + await born("goal-t"); + await turnOf("goal-t", `Reply with exactly this token and nothing else: ${token}`); + const t = await recordOf("goal-t"); + const tSpend = t?.spend ?? null; + evidence.push(`t: ${JSON.stringify(tSpend)} took ${t?.endedAt != null && t.at != null ? t.endedAt - t.at : "?"} ms`); + if (t === undefined) failures.push("t: the turn left no record"); + else if (t.error !== null) failures.push(`t: the turn failed: ${t.error}`); + if (tSpend === null) failures.push("t: the completed turn's record has no spend"); + else { + if (tSpend.model === null) failures.push("t: the spend names no model"); + if ((tSpend.inputTokens ?? 0) + (tSpend.outputTokens ?? 0) <= 0) failures.push(`t: no tokens recorded (${tSpend.inputTokens} in, ${tSpend.outputTokens} out)`); + if (!(typeof tSpend.costUsd === "number" && tSpend.costUsd > 0)) failures.push(`t: no charge recorded on an API-key run (costUsd ${tSpend.costUsd})`); + } + if (t !== undefined && (t.endedAt === null || t.at === null || t.endedAt < t.at)) failures.push(`t: endedAt ${t.endedAt} is not at or after the start ${t.at}`); + + // s — a turn stopped mid-run. + await born("goal-s"); + const stop = new AbortController(); + const timer = setTimeout(() => stop.abort(new Error("stopped by the goal check")), STOP_AFTER_MS); + await turnOf("goal-s", "Count from 1 to 3000, one number per line, with no other text.", stop.signal).catch(() => undefined); + clearTimeout(timer); + const s = await recordOf("goal-s"); + evidence.push(`s: stopped=${s?.stopped} error=${JSON.stringify(s?.error)} spend=${JSON.stringify(s?.spend ?? null)}`); + if (s === undefined) failures.push("s: the stopped turn left no record"); + else if (s.spend === null) failures.push("s: the stopped turn's record has no spend"); + else if (s.spend.harness !== "claude-code") failures.push(`s: the stopped turn's spend names ${s.spend.harness}, not claude-code`); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } + return { failures, evidence: evidence.join(" · ") }; +}); diff --git a/packages/harness-manager/README.md b/packages/harness-manager/README.md index f78920d8c9..251bdabd0a 100644 --- a/packages/harness-manager/README.md +++ b/packages/harness-manager/README.md @@ -256,14 +256,14 @@ export const claudeCodeSession = harnessSessionFlow({ }); ``` -`agent` builds the harness block from the feeds the flow hands it: `cwd`, `resume`, `onSession`, `model`, `worktree` and `onSteerable`. These are the session flow's own feeds. It has no `onUsage` or `onActivity`, which feed the manager's task records. `workspace` says where each session works: `provision` makes a session's directory on its first turn, `locate` finds it without making it, and `branch` names the branch whose pull requests the flow checks when a turn ends (`null` skips the check). A session is keyed by its flow kind, tenant and id, never by anything the caller sent. +`agent` builds the harness block from the feeds the flow hands it: `cwd`, `resume`, `onSession`, `model`, `worktree`, `onSteerable` and `onUsage`. These are the session flow's own feeds. `onUsage` records a subscription window the harness hears about during a run in the user's [usage record](#subscription-usage), as the manager's does; the flow's `harnessUsage` action reads it, with the readers you pass as `usageReaders` (none by default). It has no `onActivity`, which feeds the manager's task records. `workspace` says where each session works: `provision` makes a session's directory on its first turn, `locate` finds it without making it, and `branch` names the branch whose pull requests the flow checks when a turn ends (`null` skips the check). A session is keyed by its flow kind, tenant and id, never by anything the caller sent. The flow handles the rest: - **Continuity.** `resume` hands the harness the conversation it last confirmed, and `onSession` is the only thing that records a new one. If the harness refuses to continue that conversation, the session stays bound to it, and each later turn fails the same way. To recover, start a new session. - **The model.** A session is created with the model it runs (`state.model`), which never changes after that. Creating one without a model is refused with `HARNESS_SESSION_MODEL_REQUIRED`. `defaultModel` is the model used for a stored session that has none recorded. - **Actions.** `turn` takes a prompt and waits for any running turn to finish first (`concurrency: "defer"`). `steer` hands a line to the running turn if its harness offered to take one, and answers `{ taken, live }`. -- **A record per turn.** `state.turns` keeps the latest 20 turns, by request id. Each record holds the conversation the turn tried to resume, the one the harness confirmed, and how the turn failed: core's failure class, the harness's error text, and whether it was stopped. Read it to show a user why a turn failed. The schema is `harnessSessionStateSchema`. +- **A record per turn.** `state.turns` keeps the latest 20 turns, by request id. Each record holds the conversation the turn tried to resume, the one the harness confirmed, and how the turn failed: core's failure class, the harness's error text, and whether it was stopped. Read it to show a user why a turn failed. When the turn ends, however it ended, its record also gets `spend`, what the turn spent as a task's spend reads (`taskSpendView`'s shape: model, tokens, and the vendor's charge when it reported one, `null` when the harness reported nothing), and `endedAt`, so the turn took `endedAt - at`. The schema is `harnessSessionStateSchema`. - **A title.** The session takes its title from its first prompt (`harnessSessionTitle`). ## What a phase supplies diff --git a/packages/harness-manager/src/session-flow.ts b/packages/harness-manager/src/session-flow.ts index 5a84599658..8c2a74a5c5 100644 --- a/packages/harness-manager/src/session-flow.ts +++ b/packages/harness-manager/src/session-flow.ts @@ -37,6 +37,13 @@ * on the turn. A red check is reported, never a failed turn: the person is * there, and the next turn is theirs. * + * When a turn ends, however it ended, the flow also records what it spent on + * its record: the request's own spend, as every harness reports it into the + * request (`reportHarnessSpend`), folded by orchestration's `taskSpendView` + * so a turn reads like a task. And a subscription window a harness hears + * about during a run (`onUsage`) lands in the person's usage record, read + * through the flow's `harnessUsage` query (FIX-2398). + * * Nothing here names a harness. A harness that needs more than these feeds has * found a gap in its own package. The flow writes no item of its own. */ @@ -51,6 +58,7 @@ import { sequencer, type HarnessSteering, } from "@flow-state-dev/core"; +import { taskSpendView, taskSpendViewSchema, type TaskSpendSource } from "@flow-state-dev/orchestration/tasks"; import { withOutcome } from "@flow-state-dev/core/helpers"; import type { HarnessBlock, @@ -59,7 +67,10 @@ import type { HarnessRunHandle, HarnessSessionHook, HarnessSteerHook, + HarnessUsageHook, + HarnessUsageReader, } from "@flow-state-dev/core/types"; +import { readRequestSpend, requestSpendSources } from "@flow-state-dev/core/types"; import { recordBranchPullRequests } from "@flow-state-dev/workspace"; import { z } from "zod"; import { @@ -68,6 +79,7 @@ import { runCommandChecks, type CommandCheckOptions, } from "./command-check"; +import { harnessUsageAction, recordHarnessUsage } from "./harness-usage"; /** What one turn asked for and what came of it, recorded under its request id. */ export const harnessTurnRecordSchema = z.object({ @@ -87,6 +99,15 @@ export const harnessTurnRecordSchema = z.object({ steered: z.array(z.string()).nullable().default(null), /** The checks `verify` ran when the turn ended, in order; `null` when none ran. */ checks: z.array(commandCheckResultSchema).nullable().default(null), + /** + * What the turn spent, as a task's spend reads (`taskSpendView`): the model, + * tokens, and the vendor's charge when it reported one. Written when the + * turn ends, however it ended; `null` before that, and on a turn that spent + * nothing. + */ + spend: taskSpendViewSchema.nullable().default(null), + /** When the turn ended, in epoch millis: its duration is `endedAt - at`. */ + endedAt: z.number().nullable().default(null), }); /** One turn's record. */ @@ -161,6 +182,12 @@ export interface HarnessSessionFeeds { * A harness that can't take a line in flight never calls it. */ readonly onSteerable: HarnessSteerHook; + /** + * Records a subscription window the harness heard about during the run in + * the person's usage record, read through the flow's `harnessUsage` query. + * A harness whose vendor sends none never calls it. + */ + readonly onUsage: HarnessUsageHook; } /** Options for {@link harnessSessionFlow}. */ @@ -182,6 +209,12 @@ export interface HarnessSessionFlowOptions { * fail the turn. Unset, nothing runs. */ readonly verify?: CommandCheckOptions; + /** + * One reader per harness for the flow's `harnessUsage` query, keyed by the + * harness's name, run only when a caller asks it to refresh. Unset, the + * query reads only what runs recorded. + */ + readonly usageReaders?: Readonly>; } /** What a `verify` resolver is handed: whose session, and where it works. */ @@ -217,6 +250,8 @@ const BLANK_TURN: HarnessTurnRecord = { at: null, steered: null, checks: null, + spend: null, + endedAt: null, }; /** `state` with the turn `requestId` changed by `change`, keeping only the latest {@link KEPT_TURNS}. */ @@ -283,6 +318,10 @@ export function harnessSessionFeeds( ): HarnessSessionFeeds { return { onSteerable: (steer, ctx) => steering.onSteerable(steer, ctx), + // The person running the turn is the principal, so the window lands in their row. + onUsage: async (report, ctx) => { + await recordHarnessUsage(ctx as never, report); + }, model: (ctx) => (ctx.session.state as HarnessSessionState).model ?? defaultModel, cwd: (ctx) => workspace.provision(placeOf(kind, ctx)), worktree: (ctx) => workspace.locate(placeOf(kind, ctx)), @@ -435,6 +474,27 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { await changeState(ctx, (state) => withTurn(state, requestId, (record) => ({ ...record, checks }))); }, }); + // When a turn ends, however it ended: what it spent, on its record. The + // harness reported its run into the request before its block returned, so + // the request's pending spend is the whole turn. A failed write is a note, + // never a failed turn. + const turnSpend = handler({ + name: "turn-spend", + execute: async (_input: unknown, ctx): Promise => { + const harnessCtx = ctx as unknown as HarnessCallbackContext; + const requestId = requestOf(harnessCtx); + if ((harnessCtx.session.state as HarnessSessionState).turns?.[requestId] === undefined) return; + const spend = turnSpendOf(readRequestSpend(ctx), requestId); + try { + await changeState(harnessCtx, (state) => + withTurn(state, requestId, (record) => ({ ...record, spend, endedAt: Date.now() })), + ); + } catch (error) { + ctx.emit.status(`What the turn spent was not recorded: ${(error as Error).message}`, { transient: false }); + } + }, + }); + const turnEnd = sequencer({ name: "turn-end" }).tap(turnSpend).tap(branchPullRequests); const turn = sequencer({ name: "turn", description: "One turn of a harness session: the prompt, run by the harness where the session works.", @@ -455,8 +515,10 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { concurrency: "defer", }, steer, + // What each harness's subscription has used, as `onUsage` recorded it. + harnessUsage: harnessUsageAction({ readers: options.usageReaders ?? {} }), }, - request: { onFinished: branchPullRequests }, + request: { onFinished: turnEnd }, session: { stateSchema: harnessSessionStateSchema, // The page reads the model, the binding and the turns. The model is the @@ -471,3 +533,24 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { }, })(); } + +/** + * A turn's spend, as a task's spend reads: its request's spend folded by + * `taskSpendView`, or `null` when it spent nothing (or the context carries no + * spend ledger). + */ +function turnSpendOf(spend: ReturnType, requestId: string) { + if (spend === undefined) return null; + const sources = requestSpendSources(spend) as TaskSpendSource[]; + if (sources.length === 0) return null; + const run = spend.harnessRuns.at(-1); + const model = Object.keys(spend.byModel).at(-1) ?? null; + return taskSpendView({ + status: "completed", + spend: { + requests: [ + { requestId, settled: sources, live: null, harness: run?.harness ?? null, harnessModel: run?.model ?? null, model, writer: null, admittedAt: null, recovered: null }, + ], + }, + }); +} diff --git a/packages/harness-manager/test/turn-spend.spec.ts b/packages/harness-manager/test/turn-spend.spec.ts new file mode 100644 index 0000000000..80e1dbeaab --- /dev/null +++ b/packages/harness-manager/test/turn-spend.spec.ts @@ -0,0 +1,129 @@ +/** + * What a harness session's turn spent, and the windows its runs heard about + * (FIX-2398), against a stub harness that reports as every harness does. + * + * Why it matters: a person working a session picks a harness by what it + * costs, so each turn's record carries the tokens and charge the vendor + * reported, however the turn ended, and a subscription window a run heard + * about is readable afterwards. Read off the session state a page reads, + * never off the harness's own report. + * + * Red states produced before these were trusted: + * - drop the turn-end spend step from `onFinished`: every `spend` here is null. + * - drop `onUsage` from the feeds: the usage query lists no window. + */ +import { DEFAULT_ORG_ID, handler } from "@flow-state-dev/core"; +import { reportHarnessSpend, type HarnessBlock, type HarnessCallbackContext } from "@flow-state-dev/core/types"; +import { createInMemoryStores, runAction } from "@flow-state-dev/engine"; +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { harnessSessionFlow, type HarnessSessionFeeds, type HarnessSessionState } from "../src"; + +type Stores = ReturnType; + +/** What the stub harness does in its run. */ +interface Script { + /** Reported as the run's spend, as a harness reports from its `finally`. Unset: it reports none. */ + spend?: { inputTokens: number; outputTokens: number; costUsd: number | null; billing: "api-key" | "subscription" }; + /** The run then throws, as a harness that failed does. */ + fail?: boolean; + /** A window heard during the run. */ + window?: boolean; +} + +const stubAgent = (script: Script) => (feeds: HarnessSessionFeeds) => + handler({ + name: "stub-harness", + inputSchema: z.object({ prompt: z.string() }), + execute: async (_input, ctx) => { + const harnessCtx = ctx as unknown as HarnessCallbackContext; + await feeds.resume(harnessCtx); + if (script.window === true) { + await feeds.onUsage( + { harness: "stub", window: { id: "five_hour", label: "5h", minutes: 300, usedPercent: 42, level: "ok", resetsAt: null }, observedAt: Date.now(), runStartedAt: Date.now() - 10, accountId: "acct" }, + harnessCtx, + ); + } + try { + if (script.fail === true) throw new Error("the run broke"); + return { outcome: "completed" }; + } finally { + if (script.spend !== undefined) reportHarnessSpend(ctx, { runId: "run-1", harness: "stub", model: "stub-model", ...script.spend }); + } + }, + }) as unknown as HarnessBlock; + +const workspace = { provision: async () => "/nowhere", locate: () => "/nowhere", branch: () => null }; + +function flowOf(script: Script) { + return harnessSessionFlow({ kind: "stub-session", workspace, agent: stubAgent(script) }); +} + +async function born(stores: Stores, sessionId: string) { + const now = Date.now(); + await stores.session.set( + sessionId, + { id: sessionId, flowKind: "stub-session", userId: "u", orgId: DEFAULT_ORG_ID, metadata: undefined, latestRequestId: undefined, state: { model: "stub-model" }, version: 0, createdAt: now, updatedAt: now, journal: [] } as never, + "any" as never, + ); +} + +async function run(stores: Stores, sessionId: string, script: Script, actionName = "turn", input: unknown = { prompt: "hello" }) { + if ((await stores.session.get(sessionId)) === undefined) await born(stores, sessionId); + return runAction({ flow: flowOf(script), actionName, input, userId: "u", orgId: DEFAULT_ORG_ID, sessionId, stores, runtimeConfig: {} } as never); +} + +/** The only turn's record, as a page reads it from session state. */ +async function onlyTurn(stores: Stores, sessionId: string) { + const state = (await stores.session.get(sessionId))?.state as HarnessSessionState; + const turns = Object.values(state.turns ?? {}); + expect(turns).toHaveLength(1); + return turns[0]!; +} + +describe("a harness session turn's spend", () => { + it("is on the turn's record when it ends: the model, the tokens and the vendor's charge, and when it ended", async () => { + const stores = createInMemoryStores(); + await run(stores, "s-paid", { spend: { inputTokens: 1200, outputTokens: 300, costUsd: 0.04, billing: "api-key" } }); + const turn = await onlyTurn(stores, "s-paid"); + expect(turn.spend).toMatchObject({ harness: "stub", model: "stub-model", inputTokens: 1200, outputTokens: 300, costUsd: 0.04, unreported: false, live: false }); + expect(turn.endedAt).not.toBeNull(); + expect(turn.endedAt!).toBeGreaterThanOrEqual(turn.at!); + }); + + it("reads no dollar figure for a subscription run, never $0", async () => { + const stores = createInMemoryStores(); + await run(stores, "s-sub", { spend: { inputTokens: 50, outputTokens: 5, costUsd: null, billing: "subscription" } }); + const turn = await onlyTurn(stores, "s-sub"); + expect(turn.spend?.inputTokens).toBe(50); + expect(turn.spend?.costUsd).toBeNull(); + }); + + it("is recorded for a turn that failed too, since its tokens were spent all the same", async () => { + const stores = createInMemoryStores(); + await run(stores, "s-failed", { fail: true, spend: { inputTokens: 700, outputTokens: 70, costUsd: 0.01, billing: "api-key" } }).catch(() => undefined); + const turn = await onlyTurn(stores, "s-failed"); + expect(turn.error).toBe("the run broke"); + expect(turn.spend).toMatchObject({ inputTokens: 700, outputTokens: 70, costUsd: 0.01 }); + }); + + it("is null for a turn whose harness reported nothing, with its end still recorded", async () => { + const stores = createInMemoryStores(); + await run(stores, "s-none", {}); + const turn = await onlyTurn(stores, "s-none"); + expect(turn.spend).toBeNull(); + expect(turn.endedAt).not.toBeNull(); + }); +}); + +describe("a window a turn's run heard about", () => { + it("is readable afterwards through the session flow's usage query", async () => { + const stores = createInMemoryStores(); + await run(stores, "s-window", { window: true }); + const result = (await run(stores, "s-window", {}, "harnessUsage", {})) as { result?: unknown; output?: unknown }; + const answer = JSON.stringify(result); + expect(answer).toContain("five_hour"); + expect(answer).toContain("42"); + }); +}); diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md index f7e1479fb6..51ea90051d 100644 --- a/packages/orchestration/README.md +++ b/packages/orchestration/README.md @@ -171,9 +171,11 @@ before the task is checked. The write is `recordSpend(id, { requestId, part, spe { claim })`, fenced by the row's identity only, so a failed attempt's tokens land even after its row settled; it emits no `task-change`. A row written before this field reads `spend: null`. `taskSpendView(task)` folds it to `{ harness, model, inputTokens, -outputTokens, unreported, live }`, the shape `listTasks` returns from +outputTokens, costUsd, unreported, live }`, the shape `listTasks` returns from `taskToolActions` (never to a model): tokens are `null` until something reports one, -never 0. If you implement `TaskCollectionRef` yourself, implement `recordSpend` too, or +never 0. `costUsd` is the sum of the charges vendors reported (an API-key harness +run), or `null` when none did; a subscription run reports none, and nothing estimates +one. If you implement `TaskCollectionRef` yourself, implement `recordSpend` too, or the board writes no spend. `rowsInSession(ledger, sessionId)` returns the rows that run in a session, in ledger diff --git a/packages/orchestration/src/tasks/schema/spend.ts b/packages/orchestration/src/tasks/schema/spend.ts index 8fe023a11d..47be708475 100644 --- a/packages/orchestration/src/tasks/schema/spend.ts +++ b/packages/orchestration/src/tasks/schema/spend.ts @@ -70,6 +70,13 @@ export const taskSpendViewSchema = z.object({ inputTokens: z.number().nullable(), /** Output tokens likewise. */ outputTokens: z.number().nullable(), + /** + * What the vendors billed, in US dollars: the sum of every charge a source + * reported (an API-key harness run), or `null` when none reported one. A + * subscription run reports none, and never reads as a cost. Never an + * estimate (FIX-2398 D3). Absent on a view stored before it existed. + */ + costUsd: z.number().nullable().optional(), /** `true` while some source in the count hasn't reported its tokens. */ unreported: z.boolean(), /** `true` while a request for the task is running; never on a terminal row. */ diff --git a/packages/orchestration/src/tasks/spend.ts b/packages/orchestration/src/tasks/spend.ts index 17503de805..a572371ac5 100644 --- a/packages/orchestration/src/tasks/spend.ts +++ b/packages/orchestration/src/tasks/spend.ts @@ -96,7 +96,8 @@ export function spendAfterWrite(task: Pick, write: TaskSpendWrite * admission that recorded nothing recovered drops its model calls and keeps * its harness runs whole. A predecessor's ending that is only late replaces its live part * with what it settles, so nothing counts twice. `live` is never true on a - * terminal row either. + * terminal row either. `costUsd` sums the charges vendors reported, over + * the same parts, and is `null` when none did (FIX-2398). */ export function taskSpendView(task: Pick & Partial>): TaskSpendView | null { if (task.spend == null) return null; @@ -104,6 +105,7 @@ export function taskSpendView(task: Pick & Partial & Partial source.unreported) reportedAny = true; if (source.unreported > 0) unreported = true; } @@ -129,6 +132,7 @@ export function taskSpendView(task: Pick & Partial entry.live !== null && !superseded(entry)), }; diff --git a/packages/orchestration/test/collection/record-spend.test.ts b/packages/orchestration/test/collection/record-spend.test.ts index 39471af2cb..bda42615f7 100644 --- a/packages/orchestration/test/collection/record-spend.test.ts +++ b/packages/orchestration/test/collection/record-spend.test.ts @@ -40,7 +40,20 @@ describe("taskSpendView", () => { it("names a harness run's model before any tokens, and reads its tokens as null, not 0, while unreported", () => { const view = taskSpendView({ spend: { requests: [{ requestId: "q1", settled: [], live: [{ source: "codex", kind: "harness", billing: null, inputTokens: 0, outputTokens: 0, costUsd: null, runs: 1, unreported: 1 }], harness: "codex", harnessModel: "gpt-5", model: null, writer: null, admittedAt: null, recovered: null }] } }); - expect(view).toEqual({ harness: "codex", model: "gpt-5", inputTokens: null, outputTokens: null, unreported: true, live: true }); + expect(view).toEqual({ harness: "codex", model: "gpt-5", inputTokens: null, outputTokens: null, costUsd: null, unreported: true, live: true }); + }); + + // A person picks a harness by what it cost, so a cost is only what a vendor billed (FIX-2398 D3): + // charges add across requests, and a subscription run, which bills none, never reads as $0. + it("sums the charges vendors reported, and reads null rather than 0 when only subscription runs ran", () => { + const source = (costUsd: number | null, billing: "api-key" | "subscription") => ({ source: "claude-code", kind: "harness" as const, billing, inputTokens: 100, outputTokens: 10, costUsd, runs: 1, unreported: 0 }); + const entry = (requestId: string, settled: ReturnType[]) => ({ requestId, settled, live: null, harness: "claude-code", harnessModel: "claude-sonnet-4-5", model: null, writer: null, admittedAt: null, recovered: null }); + const subscriptionOnly = taskSpendView({ status: "completed", spend: { requests: [entry("q1", [source(null, "subscription")])] } }); + expect(subscriptionOnly?.costUsd).toBeNull(); + expect(subscriptionOnly?.inputTokens).toBe(100); + const mixed = taskSpendView({ status: "completed", spend: { requests: [entry("q1", [source(null, "subscription")]), entry("q2", [source(0.25, "api-key")]), entry("q3", [source(0.5, "api-key")])] } }); + expect(mixed?.costUsd).toBeCloseTo(0.75); + expect(mixed?.inputTokens).toBe(300); }); }); @@ -92,7 +105,7 @@ describe.each([ const before = events.length; await collection.recordSpend!("t", { requestId: "q1", part: "live", spend: harnessRun(null, null) }, { claim }); - expect(view(collection)).toEqual({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: null, outputTokens: null, unreported: true, live: true }); + expect(view(collection)).toEqual({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: null, outputTokens: null, costUsd: null, unreported: true, live: true }); await collection.recordSpend!("t", { requestId: "q1", part: "live", spend: harnessRun(100, 20) }, { claim }); expect(view(collection)).toMatchObject({ inputTokens: 100, outputTokens: 20, unreported: false, live: true }); @@ -122,7 +135,7 @@ describe.each([ // And on a completed row, a late ending of the first request still lands. await collection.recordSpend!("t", { requestId: "q1", part: "settle", spend: modelCalls(4, 4) }, { claim }); - expect(view(collection)).toEqual({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 1254, outputTokens: 254, unreported: false, live: false }); + expect(view(collection)).toEqual({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 1254, outputTokens: 254, costUsd: null, unreported: false, live: false }); }); it("keeps a dead request's live tokens but stops reading live once a retry finishes the task", async () => { diff --git a/packages/orchestration/test/skills/list-tasks-spend.test.ts b/packages/orchestration/test/skills/list-tasks-spend.test.ts index d714c89233..08275beef7 100644 --- a/packages/orchestration/test/skills/list-tasks-spend.test.ts +++ b/packages/orchestration/test/skills/list-tasks-spend.test.ts @@ -40,7 +40,7 @@ describe("listTasks and a task's spend", () => { const caller = { ...(ctx as object), user: { identity: { userId: "u1" } } }; const acted = (await runForTest(actions["listTasks_desk"]!.block as never, {} as never, caller as never)) as { tasks: Array> }; const byId = Object.fromEntries(acted.tasks.map((t) => [t.id, t])); - expect(byId[claimed.id]!.spend).toEqual({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 11_200, outputTokens: 1_240, unreported: false, live: true }); + expect(byId[claimed.id]!.spend).toEqual({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 11_200, outputTokens: 1_240, costUsd: null, unreported: false, live: true }); const other = claimed.id === "t_ran" ? "t_waits" : "t_ran"; expect(byId[other]!.spend).toBeNull(); diff --git a/packages/orchestration/test/task-board/task-spend-run.test.ts b/packages/orchestration/test/task-board/task-spend-run.test.ts index e996710d3c..e840cb4e70 100644 --- a/packages/orchestration/test/task-board/task-spend-run.test.ts +++ b/packages/orchestration/test/task-board/task-spend-run.test.ts @@ -106,7 +106,7 @@ describe("a task's run writes its spend onto the row", () => { const running = t.run(); await t.reached[0]!.opened; - expect(await t.viewAfterTick()).toEqual({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: null, outputTokens: null, unreported: true, live: true }); + expect(await t.viewAfterTick()).toEqual({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: null, outputTokens: null, costUsd: null, unreported: true, live: true }); t.steps[0]!.open(); await t.reached[1]!.opened; expect(await t.viewAfterTick()).toMatchObject({ inputTokens: 100, outputTokens: 20, live: true }); @@ -117,7 +117,7 @@ describe("a task's run writes its spend onto the row", () => { const result = await running; expect(result.error).toBeUndefined(); - expect(await t.view()).toEqual({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 1000, outputTokens: 200, unreported: false, live: false }); + expect(await t.view()).toEqual({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 1000, outputTokens: 200, costUsd: null, unreported: false, live: false }); }, 20_000); it("keeps what a cancelled run spent, and stops reading it as live", async () => { diff --git a/packages/ui/README.md b/packages/ui/README.md index 4412344596..29e70449d6 100644 --- a/packages/ui/README.md +++ b/packages/ui/README.md @@ -57,6 +57,7 @@ Generic, framework-agnostic components. No dependency on `@flow-state-dev/*`. | `selection` | Single (enum) or multi (array of enum) choice card for a `user_input` suspension | | `form` | Flat-object form card for a `user_input` suspension — one control per scalar/enum property | | `model-badge` | Small pill showing the `ModelIdentity` that answered. Renders nothing when the model is undefined | +| `spend` | One line for what a run spent: model, tokens, the vendor's charge when it reported one, and optionally attempts and duration. Reads a task's or a harness turn's folded spend | | `audit-annotation` | Compact card for `responseAuditor` findings: severity indicators, per-analyzer scores, evidence. Fed by the `audit-annotation` component item | ## Framework Integration diff --git a/packages/ui/registry.json b/packages/ui/registry.json index b8e988b980..66bfdc7eec 100644 --- a/packages/ui/registry.json +++ b/packages/ui/registry.json @@ -336,6 +336,20 @@ }], "categories": ["ai", "framework"] }, + { + "name": "spend", + "title": "Spend", + "type": "registry:component", + "description": "One line for what a run spent: model, tokens, the vendor's charge when it reported one, and optionally attempts and duration. Reads a task row's or a harness turn's folded spend; a count nobody reported reads as such, never as 0, and a subscription run shows no dollar figure.", + "dependencies": [], + "registryDependencies": [], + "files": [{ + "type": "registry:component", + "path": "registry/components/spend.tsx", + "target": "components/flow-state/spend.tsx" + }], + "categories": ["ai", "framework"] + }, { "name": "approval", "title": "Approval", diff --git a/packages/ui/registry/components/spend.tsx b/packages/ui/registry/components/spend.tsx new file mode 100644 index 0000000000..95d61af25b --- /dev/null +++ b/packages/ui/registry/components/spend.tsx @@ -0,0 +1,92 @@ +"use client"; + +import { cn } from "@/lib/utils"; + +/* + * What a run spent, as one line: the model, the tokens, the vendor's charge + * when it reported one, and optionally the attempts and how long it took. The + * numbers are read from what the server folded (a task row's or a harness + * turn's `spend`), never summed here. A count nobody reported reads as such, + * never as 0; one still missing a source is marked partial. A run billed by + * subscription reports no charge and shows none: there is no dollar figure + * to show, and an estimate would be one nobody billed. + */ + +/** + * What the line reads: the fields of orchestration's `TaskSpendView`, which a + * task row and a harness session's turn both carry. `costUsd` absent reads as + * none reported. + */ +export interface SpendSummary { + harness: string | null; + model: string | null; + inputTokens: number | null; + outputTokens: number | null; + costUsd?: number | null; + unreported: boolean; + live: boolean; +} + +/** What else the line can say. */ +export interface SpendLineOptions { + /** Input and output apart ("10.1k in · 2.3k out") rather than one total. */ + split?: boolean; + /** The harness's name, first, for a view with no harness tag of its own. */ + harnessLabel?: string; + /** The task's attempts; shown from the second on. */ + attempts?: number; + /** How long it took, in ms. */ + durationMs?: number | null; +} + +/** 980, 12.4k, 1.2M. */ +export function tokenCount(n: number): string { + if (n < 1000) return String(n); + if (n < 1_000_000) return `${(n / 1000).toFixed(1)}k`; + return `${(n / 1_000_000).toFixed(1)}M`; +} + +/** $0.0042, $0.42, $12.40. */ +export function costAmount(usd: number): string { + return usd < 0.01 ? `$${usd.toFixed(4)}` : `$${usd.toFixed(2)}`; +} + +/** 42s, 4m 12s, 1h 3m. */ +export function durationText(ms: number): string { + const seconds = Math.max(0, Math.round(ms / 1000)); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ${seconds % 60}s`; + return `${Math.floor(minutes / 60)}h ${minutes % 60}m`; +} + +/** + * "claude-sonnet-4-5 · 12.4k tokens · $0.42 · 2 attempts · 4m 12s". Empty + * when there is no spend to show. + */ +export function spendLine(spend: SpendSummary | null | undefined, options: SpendLineOptions = {}): string { + if (spend == null) return ""; + const partial = spend.unreported ? " (partial)" : ""; + const side = (n: number | null) => (n === null ? "not reported" : tokenCount(n)); + const tokens = + spend.inputTokens === null && spend.outputTokens === null + ? "no tokens reported yet" + : options.split === true + ? `${side(spend.inputTokens)} in · ${side(spend.outputTokens)} out${partial}` + : `${tokenCount((spend.inputTokens ?? 0) + (spend.outputTokens ?? 0))} tokens${partial}`; + const cost = spend.costUsd == null ? null : costAmount(spend.costUsd); + const attempts = options.attempts !== undefined && options.attempts > 1 ? `${options.attempts} attempts` : null; + const duration = options.durationMs == null ? null : durationText(options.durationMs); + return [options.harnessLabel, spend.model, tokens, cost, attempts, duration].filter(Boolean).join(" · "); +} + +/** The spend line, or nothing when there is no spend to show. */ +export function Spend({ spend, className, ...options }: SpendLineOptions & { spend: SpendSummary | null | undefined; className?: string }) { + const line = spendLine(spend, options); + if (!line) return null; + return ( + + {line} + + ); +} diff --git a/packages/ui/test/render/spend.test.tsx b/packages/ui/test/render/spend.test.tsx new file mode 100644 index 0000000000..8fcd1927ec --- /dev/null +++ b/packages/ui/test/render/spend.test.tsx @@ -0,0 +1,68 @@ +// @vitest-environment happy-dom +/** + * The spend line every app draws for a task or a harness turn (FIX-2398). + * + * Why it matters: a person picks a harness by what it cost, so the line must + * say the vendor's charge when there is one and never invent one: a + * subscription run, which bills nothing per run, shows no dollar figure, and + * a count nobody reported reads as such rather than 0. + * + * Red states produced before these were trusted: format a `null` cost as + * `$0.00`: the subscription case fails; drop the attempts guard: a + * first-attempt task reads "1 attempts". + */ +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import { Spend, durationText, spendLine, type SpendSummary } from "../../registry/components/spend"; + +afterEach(cleanup); + +const view = (over: Partial = {}): SpendSummary => ({ + harness: "claude-code", + model: "claude-sonnet-4-5", + inputTokens: 10_100, + outputTokens: 2_300, + costUsd: 0.42, + unreported: false, + live: false, + ...over, +}); + +describe("spendLine", () => { + it("says the model, the tokens and the vendor's charge", () => { + expect(spendLine(view())).toBe("claude-sonnet-4-5 · 12.4k tokens · $0.42"); + expect(spendLine(view(), { split: true, harnessLabel: "Claude Code" })).toBe("Claude Code · claude-sonnet-4-5 · 10.1k in · 2.3k out · $0.42"); + }); + + it("shows no dollar figure for a run that reported no charge, never $0.00", () => { + expect(spendLine(view({ costUsd: null }))).toBe("claude-sonnet-4-5 · 12.4k tokens"); + expect(spendLine(view({ costUsd: undefined }))).not.toContain("$"); + }); + + it("reads an unreported count as such, and marks a partial one", () => { + expect(spendLine(view({ inputTokens: null, outputTokens: null, costUsd: null }))).toBe("claude-sonnet-4-5 · no tokens reported yet"); + expect(spendLine(view({ unreported: true }))).toContain("12.4k tokens (partial)"); + }); + + it("counts attempts only from the second on, and says how long it took", () => { + expect(spendLine(view(), { attempts: 1 })).not.toContain("attempt"); + expect(spendLine(view(), { attempts: 3, durationMs: 252_000 })).toBe("claude-sonnet-4-5 · 12.4k tokens · $0.42 · 3 attempts · 4m 12s"); + expect(durationText(42_400)).toBe("42s"); + expect(durationText(3_780_000)).toBe("1h 3m"); + }); + + it("is empty when there is no spend to show", () => { + expect(spendLine(null)).toBe(""); + expect(spendLine(undefined)).toBe(""); + }); +}); + +describe("Spend", () => { + it("draws the line, marked live while the run works, and nothing without spend", () => { + const { container } = render(); + expect(screen.getByText("claude-sonnet-4-5 · 12.4k tokens · $0.42").getAttribute("data-spend")).toBe("live"); + cleanup(); + expect(render().container.innerHTML).toBe(""); + void container; + }); +}); From 20874f9e6522a82009937cc123a7333809df0dff Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:16:46 +0000 Subject: [PATCH 05/24] test(kitchen-sink): the turn record fixture names checks Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- apps/kitchen-sink/test/harness-turn.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/kitchen-sink/test/harness-turn.test.ts b/apps/kitchen-sink/test/harness-turn.test.ts index 9dbce3b8eb..f3c94565e0 100644 --- a/apps/kitchen-sink/test/harness-turn.test.ts +++ b/apps/kitchen-sink/test/harness-turn.test.ts @@ -26,7 +26,7 @@ import { type PlanPanelInput, } from "@/lib/harness-turn"; -const blank: HarnessTurnRecord = { asked: null, confirmed: null, failure: null, stopped: null, error: null, at: 1, steered: null, spend: null, endedAt: null }; +const blank: HarnessTurnRecord = { asked: null, confirmed: null, failure: null, stopped: null, error: null, at: 1, steered: null, checks: null, spend: null, endedAt: null }; describe("turnNotice", () => { it("says nothing while the turn runs", () => { From 95155653b5b30acb8ee11c452285b733c95e445c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:29:02 +0000 Subject: [PATCH 06/24] fix(harness-manager): write a turn's spend inside the turn, before the request publishes its end (FIX-2398) onFinished runs after runAction emits the terminal status, so a page that refreshes on the end could read the turn before its spend landed (Codex, spec PR #4398). The spend now lands in a tap after the harness step, or in recordFailure's write for a turn that threw. A test captures session state when request.completed is persisted. The goal's control is now main. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- .../a-turn-records-what-it-spent/goal.md | 5 +-- .../a-turn-records-what-it-spent/run.mts | 13 ++++---- packages/harness-manager/src/session-flow.ts | 33 +++++++++++-------- .../harness-manager/test/turn-spend.spec.ts | 19 +++++++++++ 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/goals/harness-manager/a-turn-records-what-it-spent/goal.md b/goals/harness-manager/a-turn-records-what-it-spent/goal.md index 58a26f1831..7170023196 100644 --- a/goals/harness-manager/a-turn-records-what-it-spent/goal.md +++ b/goals/harness-manager/a-turn-records-what-it-spent/goal.md @@ -17,12 +17,13 @@ **Run:** `pnpm tsx goals/harness-manager/a-turn-records-what-it-spent/run.mts` -**Controls:** on the same command. +**Controls:** -- `GOAL_CONTROL=no-turn-spend`: the flow's turn-end step is dropped, as a flow that never writes a turn's spend would. **t** and **s** FAIL on `spend` being null. +- Today's `main`: the same `run.mts` copied into a worktree of `main` and run there. That flow writes no turn spend, so **t** and **s** FAIL on `spend` being missing. The flow writes the spend inside its turn, before the request publishes its end, so no switch from outside the package can turn it off; the baseline is the control. ## Verdict log | Date | Commit | Model | Verdict | Notes | |------|--------|-------|---------|-------| | 2026-10-31 | implementation branch, on 45cefc5 | claude-haiku-4-5 (API key) | **PASS** | Linux container. **t**: 17,634 in · 73 out · $0.022405, 4.0 s. **s**: stopped, 17,491 in · 4 out, no charge reported for the stopped run. Control `no-turn-spend` FAILED **t** (no spend, no `endedAt`) and **s** (no spend) | | 2026-10-31 | after merging FIX-2387's branch (with FIX-2384) | claude-haiku-4-5 (API key) | **PASS** | Rerun on the stacked branch. **t**: 17,487 in · 58 out · $0.0221, 3.2 s. **s**: stopped, 17,491 in · 6 out. Control `no-turn-spend` FAILED **t** and **s** again | +| 2026-10-31 | spend written inside the turn (after Codex's ordering finding) | claude-haiku-4-5 (API key) | **PASS** | **t**: 17,489 in · 68 out · $0.0088, 3.1 s. **s**: stopped, 17,493 in · 4 out. The `no-turn-spend` control above no longer bites (it dropped `onFinished`, which no longer writes), so it is retired. Control is now `main` 675f614 in a worktree: **FAILED** **t** (no spend, no `endedAt`) and **s** (no spend) | diff --git a/goals/harness-manager/a-turn-records-what-it-spent/run.mts b/goals/harness-manager/a-turn-records-what-it-spent/run.mts index 6f3299fddc..767e7addbd 100644 --- a/goals/harness-manager/a-turn-records-what-it-spent/run.mts +++ b/goals/harness-manager/a-turn-records-what-it-spent/run.mts @@ -12,9 +12,10 @@ * charge above zero, and `endedAt` is at or after the turn's start. * - **s**: a turn stopped mid-run still has `spend`, naming Claude Code. * - * Control: `GOAL_CONTROL=no-turn-spend` drops the flow's turn-end step, as a - * flow that never writes a turn's spend would. **t** and **s** FAIL on - * `spend` being null. + * Control: the same script run from a worktree of `main`, whose flow writes + * no turn spend. **t** and **s** FAIL on `spend` being missing. The flow + * writes the spend inside its turn, where nothing outside the package can + * switch it off, so the baseline is the control. * * Run: pnpm tsx goals/harness-manager/a-turn-records-what-it-spent/run.mts */ @@ -28,7 +29,6 @@ import { createInMemoryStores, runAction } from "@flow-state-dev/engine"; import { harnessSessionFlow, type HarnessSessionState, type HarnessTurnRecord } from "@flow-state-dev/harness-manager"; import { runGoal } from "../../lib/index.mts"; -const control = process.env.GOAL_CONTROL ?? ""; if ((process.env.ANTHROPIC_API_KEY ?? "") === "" && (process.env.MY_ANTHROPIC_API_KEY ?? "") !== "") { process.env.ANTHROPIC_API_KEY = process.env.MY_ANTHROPIC_API_KEY; } @@ -54,7 +54,6 @@ await runGoal(async (failures) => { detached: true, } as never) as never, }); - if (control === "no-turn-spend") (flow as { request?: unknown }).request = undefined; const stores = createInMemoryStores(); const born = async (sessionId: string) => { @@ -89,7 +88,7 @@ await runGoal(async (failures) => { if ((tSpend.inputTokens ?? 0) + (tSpend.outputTokens ?? 0) <= 0) failures.push(`t: no tokens recorded (${tSpend.inputTokens} in, ${tSpend.outputTokens} out)`); if (!(typeof tSpend.costUsd === "number" && tSpend.costUsd > 0)) failures.push(`t: no charge recorded on an API-key run (costUsd ${tSpend.costUsd})`); } - if (t !== undefined && (t.endedAt === null || t.at === null || t.endedAt < t.at)) failures.push(`t: endedAt ${t.endedAt} is not at or after the start ${t.at}`); + if (t !== undefined && (t.endedAt == null || t.at == null || t.endedAt < t.at)) failures.push(`t: endedAt ${t.endedAt} is not at or after the start ${t.at}`); // s — a turn stopped mid-run. await born("goal-s"); @@ -100,7 +99,7 @@ await runGoal(async (failures) => { const s = await recordOf("goal-s"); evidence.push(`s: stopped=${s?.stopped} error=${JSON.stringify(s?.error)} spend=${JSON.stringify(s?.spend ?? null)}`); if (s === undefined) failures.push("s: the stopped turn left no record"); - else if (s.spend === null) failures.push("s: the stopped turn's record has no spend"); + else if (s.spend == null) failures.push("s: the stopped turn's record has no spend"); else if (s.spend.harness !== "claude-code") failures.push(`s: the stopped turn's spend names ${s.spend.harness}, not claude-code`); } finally { rmSync(scratch, { recursive: true, force: true }); diff --git a/packages/harness-manager/src/session-flow.ts b/packages/harness-manager/src/session-flow.ts index 14d61e1e16..44e327b5b0 100644 --- a/packages/harness-manager/src/session-flow.ts +++ b/packages/harness-manager/src/session-flow.ts @@ -43,8 +43,9 @@ * on the turn. A red check is reported, never a failed turn: the person is * there, and the next turn is theirs. * - * When a turn ends, however it ended, the flow also records what it spent on - * its record: the request's own spend, as every harness reports it into the + * When the harness step ends, however it ended, the flow also records what + * the turn spent on its record, inside the turn so it lands before the + * request publishes its end: the request's own spend, as every harness reports it into the * request (`reportHarnessSpend`), folded by orchestration's `taskSpendView` * so a turn reads like a task. And a subscription window a harness hears * about during a run (`onUsage`) lands in the person's usage record, read @@ -439,6 +440,7 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { failure: harnessFailureKind(error), stopped: ctx.signal.aborted, error: error.message, + ...turnEndOf(ctx, requestId), })), ); throw error; @@ -521,21 +523,19 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { await changeState(harnessCtx, (state) => withTurn(state, requestId, (record) => ({ ...record, head }))); }, }); - // When a turn ends, however it ended: what it spent, on its record. The + // When the harness step returns: what the turn spent, on its record. The // harness reported its run into the request before its block returned, so - // the request's pending spend is the whole turn. A failed write is a note, - // never a failed turn. + // the request's pending spend is the whole turn. Written inside the turn, + // not at `onFinished`, so it lands before the request publishes its end and + // a page that refreshes on that end reads it. A turn that threw records it + // in `recordFailure`'s write instead. A failed write is a note, never a + // failed turn. const turnSpend = handler({ name: "turn-spend", - execute: async (_input: unknown, ctx): Promise => { - const harnessCtx = ctx as unknown as HarnessCallbackContext; - const requestId = requestOf(harnessCtx); - if ((harnessCtx.session.state as HarnessSessionState).turns?.[requestId] === undefined) return; - const spend = turnSpendOf(readRequestSpend(ctx), requestId); + execute: async (_handle: HarnessRunHandle, ctx): Promise => { + const requestId = ctx.request.identity.id; try { - await changeState(harnessCtx, (state) => - withTurn(state, requestId, (record) => ({ ...record, spend, endedAt: Date.now() })), - ); + await changeState(ctx, (state) => withTurn(state, requestId, (record) => ({ ...record, ...turnEndOf(ctx, requestId) }))); } catch (error) { ctx.emit.status(`What the turn spent was not recorded: ${(error as Error).message}`, { transient: false }); } @@ -543,7 +543,6 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { }); const turnEnd = sequencer({ name: "turn-end", inputSchema: z.unknown() }) .tap(recordTurnHead) - .tap(turnSpend) .tap(branchPullRequests); // What one turn's commits changed, between the commits its record keeps. A // read: answered in the response and stored nowhere, so an opened patch @@ -591,6 +590,7 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { .tap(titleFromPrompt) .step(agent.rescue([{ block: recordFailure }])) .tap(recordFailedOutcome) + .tap(turnSpend) .tap(verifyTurn); return defineFlow({ @@ -623,6 +623,11 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { })(); } +/** What a turn's record gets when its harness step ends: what it spent, and when. */ +function turnEndOf(ctx: object, requestId: string): { spend: ReturnType; endedAt: number } { + return { spend: turnSpendOf(readRequestSpend(ctx), requestId), endedAt: Date.now() }; +} + /** * A turn's spend, as a task's spend reads: its request's spend folded by * `taskSpendView`, or `null` when it spent nothing (or the context carries no diff --git a/packages/harness-manager/test/turn-spend.spec.ts b/packages/harness-manager/test/turn-spend.spec.ts index 80e1dbeaab..cd033ce2fa 100644 --- a/packages/harness-manager/test/turn-spend.spec.ts +++ b/packages/harness-manager/test/turn-spend.spec.ts @@ -11,6 +11,8 @@ * Red states produced before these were trusted: * - drop the turn-end spend step from `onFinished`: every `spend` here is null. * - drop `onUsage` from the feeds: the usage query lists no window. + * - write the spend from the flow's `onFinished` instead: the request's + * terminal status is published before the spend lands. */ import { DEFAULT_ORG_ID, handler } from "@flow-state-dev/core"; import { reportHarnessSpend, type HarnessBlock, type HarnessCallbackContext } from "@flow-state-dev/core/types"; @@ -115,6 +117,23 @@ describe("a harness session turn's spend", () => { expect(turn.spend).toBeNull(); expect(turn.endedAt).not.toBeNull(); }); + // A page refreshes the session when it sees the turn end, so the spend must + // be stored by the time the request publishes its terminal status. + it("is stored before the request publishes that it completed", async () => { + const stores = createInMemoryStores(); + await born(stores, "s-order"); + const seen: Array> = []; + const persist = stores.request.persistEvents.bind(stores.request); + stores.request.persistEvents = (requestId, events) => { + if (events.some((event) => (event as { type?: string }).type === "request.completed")) { + seen.push(stores.session.get("s-order").then((session) => Object.values((session?.state as HarnessSessionState).turns ?? {})[0]?.spend ?? null)); + } + return persist(requestId, events); + }; + await run(stores, "s-order", { spend: { inputTokens: 10, outputTokens: 2, costUsd: 0.001, billing: "api-key" } }); + expect(seen).toHaveLength(1); + expect(await seen[0]).toMatchObject({ inputTokens: 10, outputTokens: 2 }); + }); }); describe("a window a turn's run heard about", () => { From 7f475598144a41d80af862059db13afcb634d836 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:10:55 +0000 Subject: [PATCH 07/24] spec(FIX-2398): name the docs page that changes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- specs/issues/FIX-2398/SPEC.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/issues/FIX-2398/SPEC.md b/specs/issues/FIX-2398/SPEC.md index 2f81ba0771..e57e3d4626 100644 --- a/specs/issues/FIX-2398/SPEC.md +++ b/specs/issues/FIX-2398/SPEC.md @@ -85,7 +85,7 @@ the request's own ledger at its end ([D2](DECISIONS.md#d2)). **Docs:** harness-manager README (the turn record's `spend` and `endedAt`, the `onUsage` feed, `usageReaders`), orchestration README (`costUsd` on the spend view), ui README (the `spend` -item). No new docs-site page: the harness session and task-board pages gain one line each. +item), and the docs site's *Flow-aware components* page gains a *Spend* section. No new page. ## The PR plan From 2f4ac2c0e332d426ca181b640915f242bd2f592f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:29:33 +0000 Subject: [PATCH 08/24] =?UTF-8?q?spec(FIX-2398):=20round=201=20=E2=80=94?= =?UTF-8?q?=20the=20turn's=20spend=20is=20written=20inside=20the=20turn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex: onFinished runs after the terminal status is published, so D2's write moves inside the turn and the goal's control becomes main. Cursor's scope notes answered in DECISIONS. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- specs/issues/FIX-2398/DECISIONS.md | 15 ++++++++++++--- specs/issues/FIX-2398/SPEC.md | 11 ++++++----- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/specs/issues/FIX-2398/DECISIONS.md b/specs/issues/FIX-2398/DECISIONS.md index f2c09d776d..084abdc490 100644 --- a/specs/issues/FIX-2398/DECISIONS.md +++ b/specs/issues/FIX-2398/DECISIONS.md @@ -39,8 +39,8 @@ option is cheap and belongs in that package. | | | |---|---| | **Instead of** | A feed callback on `HarnessSessionFeeds` the harness calls with its usage; or a sum over the turn's items | -| **Because** | Every harness already reports its run into the request's ledger, from its `finally`, however the run ends. The session flow's request `onFinished` runs after the harness block returns and before the ending hands the ledger on, so `readRequestSpend(ctx)` there is the whole turn: a failed or stopped turn included. Folding it with `taskSpendView` gives a turn the same view a task has, so one component draws both (tenet 5: one fold). A feed callback would be a second report path beside `reportHarnessSpend`, kept in step by every harness | -| **Locks in** | A turn's count is final, not live: it appears when the turn ends. The record's `spend` has the view's shape, so a change to the view changes both readers at once | +| **Because** | Every harness already reports its run into the request's ledger, from its `finally`, however the run ends. So `readRequestSpend(ctx)` read right after the harness step, inside the turn, is the whole turn: in a step after it for a turn that returned, and in the failure record's own write for one that threw or was stopped. Inside the turn, because the request publishes its terminal status *before* `onFinished` runs, and a page refreshes on that status: a write at `onFinished` would land after the page read. Folding it with `taskSpendView` gives a turn the same view a task has, so one component draws both (tenet 5: one fold). A feed callback would be a second report path beside `reportHarnessSpend`, kept in step by every harness | +| **Locks in** | A turn's count is final, not live: it appears when the turn ends, never later than the request's end. Turn-card spend never reads the `harnessUsage` query, which holds subscription windows only. The record's `spend` has the view's shape, so a change to the view changes both readers at once | **What would change my mind:** a page that must show a turn's tokens rising while it runs. Then the turn writes its live part on a timer, as a task's request does (FIX-2288 D2). @@ -67,6 +67,8 @@ the turn writes its live part on a timer, as a task's request does (FIX-2288 D2) is invisible. - **Kitchen Sink installs the byte copy in PR 1**: its drift check requires every registry file. - **A turn-end write that fails is a note, never a failed turn**, like the pull-request check. +- **`costUsd` sums only sources whose `costUsd` is non-null**, the same rule as a worker's usage rows (FIX-2145), so the two never disagree. +- **Two deliverables in PR 1, one code path each:** (A) a task's and a turn's spend, through `taskSpendView`; (B) subscription windows, through `onUsage` and `harnessUsage`. They share no code. ## Considered and dropped @@ -74,7 +76,10 @@ the turn writes its live part on a timer, as a task's request does (FIX-2288 D2) |---|---| | Keep each lab's own spend line | Four copies of one line, which ER-3 says go to the package | | Put `costUsd` on FIX-2379's row view only | The row view carries the spend view (ER-4); the cost belongs in the fold it carries | -| Write the turn spend from a request-end hook | It runs after the request is over, so a page that refreshes on the turn's end can read before it lands; `onFinished` runs after the harness reported and before the request ends | +| Write the turn spend from a request-end hook, or at `onFinished` | Both run after the request publishes its terminal status, so a page that refreshes on it reads the turn before its spend lands (Codex, round 1). A request-end hook would also hold back every turn's status | +| Lift `Spend` and delete Claude Code-style's copy in PR 1 | Claude Code-style's copy takes its own row type and harness labels across six components; swapping it is app work, and the epic's D3 puts app changes after the package slice | +| Cursor's `onActivity` as its own issue | One line in `labs/teammates/host.mts`, which PR 4 already edits in its merge order; a separate issue adds a PR on the same file | +| Readers only on a host flow, as Shift Manager does | The collection has to be registered on the session flow anyway, for `onUsage` to write it. The flow's `harnessUsage` is `harnessUsageAction` itself, so the refresh and claim rules are one; a host with another flow may still read there | ## Analysis @@ -87,3 +92,7 @@ down after PR 5. - **Drafted (Oct 31)** from the issue and epic FIX-2369's seams, with the Codex and Cursor wire types read to settle D1. +- **Round 1 (Oct 31):** Codex showed `onFinished` runs after the terminal status is published + (`runAction.ts`), so D2's write moved inside the turn; the goal's control became `main`, since + no outside switch reaches a write inside the turn. Cursor's scope notes are answered under + *Considered and dropped* and *Decided, not asked*. diff --git a/specs/issues/FIX-2398/SPEC.md b/specs/issues/FIX-2398/SPEC.md index e57e3d4626..58d0eba807 100644 --- a/specs/issues/FIX-2398/SPEC.md +++ b/specs/issues/FIX-2398/SPEC.md @@ -34,7 +34,7 @@ flowchart LR T -->|"tokens > 0 and the vendor's cost, model named"| P["PASS · goal met"] I --> U["a stopped turn"] U -->|"its tokens recorded too"| P - C["control · no-turn-spend"] -.-> T + C["control · today's main"] -.-> T T -.->|"under the control"| F["must FAIL · turn spend is null"] ``` @@ -47,7 +47,7 @@ harness's handle. | **Signal** | **t**: after a completed turn, its record's `spend` names the model, `inputTokens + outputTokens > 0`, `costUsd > 0`, and `endedAt ≥ at`. **s**: a stopped turn's record carries `spend` too | | **Input** | A one-line prompt with a fresh token to echo. Any other prompt must pass | | **Anti-game** | Read the session state the page reads. Don't assert on `HarnessRunSpend` or a mocked ledger: both pass while the turn record stays empty | -| **Control that must fail** | `GOAL_CONTROL=no-turn-spend` (the flow skips its turn-end write). **t** and **s** FAIL on `spend` being `null`. Today's `main` fails the same way | +| **Control that must fail** | Today's `main`: the same script run from a worktree of `main`. **t** and **s** FAIL on `spend` being missing. The write is inside the turn ([D2](DECISIONS.md#d2)), so no outside switch can turn it off | ## What changes @@ -55,7 +55,7 @@ harness's handle. flowchart LR H["claude-code · codex · cursor
reportHarnessSpend (unchanged)"] --> L["the request's spend ledger"] L -->|"task runs (FIX-2288)"| R["task row spend"] - L -->|"turn end, new"| TR["turn record spend + endedAt"] + L -->|"harness step ends, new"| TR["turn record spend + endedAt"] R --> V["taskSpendView · + costUsd"] TR --> V V --> S["registry Spend · one line"] @@ -64,7 +64,8 @@ flowchart LR ``` One fold (`taskSpendView`) for a task and a turn, one component for every app. A turn's spend is -the request's own ledger at its end ([D2](DECISIONS.md#d2)). +the request's own ledger when its harness step ends, written inside the turn so it lands before +the request says it is done ([D2](DECISIONS.md#d2)). **A host, after:** @@ -151,6 +152,6 @@ wrong: a fifth spend line someone keeps in step. 3. **[D3](DECISIONS.md#d3) · A cost is the vendor's charge or nothing.** If wrong: a subscription run reads `$0.00`, or an estimate nobody billed. -**Open: none.** Number 1 is the one to weigh. +**Open: none.** Of the three, number 1 deserves the closest look. Feature · `orchestration` + `harness-manager` + `ui` registry, then four app PRs · small package slice · epic FIX-2369 From 6143c18401ae61df16aef28ae80fdf4a2a08e802 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:49:30 +0000 Subject: [PATCH 09/24] FIX-2398: registry drift count is 40 with FIX-2387's run-checks copy Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- apps/kitchen-sink/test/registry-drift.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/kitchen-sink/test/registry-drift.test.ts b/apps/kitchen-sink/test/registry-drift.test.ts index 1ce309485d..c725c1647c 100644 --- a/apps/kitchen-sink/test/registry-drift.test.ts +++ b/apps/kitchen-sink/test/registry-drift.test.ts @@ -37,7 +37,7 @@ describe("kitchen-sink registry drift (FIX-1477 V8)", () => { // files outside `generative/`) are installed here. If that count moves, // this test's scope moved with it and needs re-reading before trusting a // green result. - expect(compared.length).toBe(39); + expect(compared.length).toBe(40); expect(mismatches).toEqual([]); }); From 7b7e8281a99bb07a36b9728ab794caa46dc70382 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:53:17 +0000 Subject: [PATCH 10/24] FIX-2398: Kitchen Sink installs FIX-2387's run-checks and FIX-2384's updated run-changes The parent branch moved both registry files without updating the byte copies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- .../components/flow-state/run-changes.tsx | 15 ++- .../components/flow-state/run-checks.tsx | 109 ++++++++++++++++++ 2 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 apps/kitchen-sink/components/flow-state/run-checks.tsx diff --git a/apps/kitchen-sink/components/flow-state/run-changes.tsx b/apps/kitchen-sink/components/flow-state/run-changes.tsx index b4848aea11..4b78a5da93 100644 --- a/apps/kitchen-sink/components/flow-state/run-changes.tsx +++ b/apps/kitchen-sink/components/flow-state/run-changes.tsx @@ -21,8 +21,12 @@ import { cn } from "@/lib/utils"; /** One file a run changed. `added`/`removed` are `null` for a binary file. */ export type RunChangesFile = { path: string; added: number | null; removed: number | null }; -/** One file's patch, as the app's read answered it. */ -export type RunChangesPatch = { patch: string; truncated: boolean }; +/** + * One file's patch, as the app's read answered it. `verified: false` means + * git couldn't read it (the checkout is gone, say): the file shows that and + * reads again when reopened, rather than claiming it has no changes. + */ +export type RunChangesPatch = { patch: string; truncated: boolean; verified?: boolean }; export interface RunChangesProps { /** The files the run changed, or `null` when they couldn't be read. */ @@ -88,7 +92,12 @@ function FileRow({ file, loadPatch }: { file: RunChangesFile; loadPatch?: RunCha if (!next || loadPatch === undefined || (state !== null && state.status !== "failed")) return; setState({ status: "loading" }); loadPatch(file.path).then( - (patch) => setState({ status: "ready", patch }), + (patch) => + setState( + patch.verified === false + ? { status: "failed", message: "git couldn't read it from the run's checkout" } + : { status: "ready", patch }, + ), (error: unknown) => setState({ status: "failed", message: error instanceof Error ? error.message : String(error) }), ); }; diff --git a/apps/kitchen-sink/components/flow-state/run-checks.tsx b/apps/kitchen-sink/components/flow-state/run-checks.tsx new file mode 100644 index 0000000000..a6d613ad97 --- /dev/null +++ b/apps/kitchen-sink/components/flow-state/run-checks.tsx @@ -0,0 +1,109 @@ +"use client"; + +/** + * A coding run's checks: "Checks 2/2 passed", red when any failed, opening + * into each command and the end of its output. + * + * Takes plain data, so it renders whatever read supplied it: harness-manager + * puts the results on a run review's stop data (`checks`), on the run record, + * and on a session turn's record. Each result is a command, how it ended, how + * long it ran and the last of its output. + * + * A red set opens by itself, so the failure is the first thing seen. Draws + * nothing for `checks: null` or an empty list: no check ran, and "0/0 + * passed" would be a claim. + */ + +import { CheckIcon, ChevronRightIcon, XIcon } from "lucide-react"; +import { useState } from "react"; +import { cn } from "@/lib/utils"; + +/** One check's result. */ +export type RunCheck = { + command: string; + exitCode: number | null; + timedOut: boolean; + skipped: boolean; + durationMs: number; + tail: string; +}; + +export interface RunChecksProps { + /** The checks the run's completion check ran, in order, or `null` when none ran. */ + checks: RunCheck[] | null; + /** Start open. Default: open when any check failed. */ + defaultOpen?: boolean; + className?: string; +} + +/** Whether a check passed: it ran, and exited 0. */ +export function runCheckPassed(check: RunCheck): boolean { + return !check.skipped && !check.timedOut && check.exitCode === 0; +} + +/** How a check ended, in a few words. */ +function ending(check: RunCheck): string { + if (check.skipped) return "skipped"; + if (check.timedOut) return "timed out"; + if (check.exitCode === null) return "not run"; + return check.exitCode === 0 ? "passed" : `exit ${check.exitCode}`; +} + +const seconds = (ms: number) => (ms < 1000 ? `${ms}ms` : `${(ms / 1000).toFixed(1)}s`); + +/** One command: whether it passed, how it ended, and its output when opened. */ +function CheckRow({ check, defaultOpen }: { check: RunCheck; defaultOpen: boolean }) { + const [open, setOpen] = useState(defaultOpen); + const passed = runCheckPassed(check); + const hasOutput = check.tail.trim() !== ""; + return ( +
  • + + {open && hasOutput ? ( +
    {check.tail}
    + ) : null} +
  • + ); +} + +/** "Checks n/m passed", opening into each command and its output. */ +export function RunChecks({ checks, defaultOpen, className }: RunChecksProps) { + const failed = (checks ?? []).filter((check) => !runCheckPassed(check)).length; + const [open, setOpen] = useState(defaultOpen ?? failed > 0); + if (checks === null || checks.length === 0) return null; + const passed = checks.length - failed; + return ( +
    0 && "border-destructive/50", className)} data-state={failed > 0 ? "failed" : "passed"}> + + {open ? ( +
      + {checks.map((check, i) => ( + + ))} +
    + ) : null} +
    + ); +} From 8c2e648ba0f8297e7b7e0bb5f63f7b586765f6d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 23:57:16 +0000 Subject: [PATCH 11/24] FIX-2398: a request's spend reads as a task's through orchestration's requestSpendView Review on #4423: the session flow no longer hand-builds a task spend entry; orchestration owns the request-to-view step. The turn-end write happens once (a record that already ended keeps its end), and onUsage needs no cast. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- packages/harness-manager/src/session-flow.ts | 46 ++++++++----------- packages/orchestration/src/tasks/index.ts | 2 +- packages/orchestration/src/tasks/spend.ts | 24 ++++++++++ .../test/collection/record-spend.test.ts | 18 ++++++++ 4 files changed, 61 insertions(+), 29 deletions(-) diff --git a/packages/harness-manager/src/session-flow.ts b/packages/harness-manager/src/session-flow.ts index 98d35d8b96..a77c5bb33d 100644 --- a/packages/harness-manager/src/session-flow.ts +++ b/packages/harness-manager/src/session-flow.ts @@ -46,7 +46,7 @@ * When the harness step ends, however it ended, the flow also records what * the turn spent on its record, inside the turn so it lands before the * request publishes its end: the request's own spend, as every harness reports it into the - * request (`reportHarnessSpend`), folded by orchestration's `taskSpendView` + * request (`reportHarnessSpend`), folded by orchestration's `requestSpendView` * so a turn reads like a task. And a subscription window a harness hears * about during a run (`onUsage`) lands in the person's usage record, read * through the flow's `harnessUsage` query (FIX-2398). @@ -65,7 +65,7 @@ import { sequencer, type HarnessSteering, } from "@flow-state-dev/core"; -import { taskSpendView, taskSpendViewSchema, type TaskSpendSource } from "@flow-state-dev/orchestration/tasks"; +import { requestSpendView, taskSpendViewSchema, type TaskSpendView } from "@flow-state-dev/orchestration/tasks"; import { withOutcome } from "@flow-state-dev/core/helpers"; import type { HarnessBlock, @@ -77,7 +77,7 @@ import type { HarnessUsageHook, HarnessUsageReader, } from "@flow-state-dev/core/types"; -import { readRequestSpend, requestSpendSources } from "@flow-state-dev/core/types"; +import { readRequestSpend } from "@flow-state-dev/core/types"; import { recordBranchPullRequests, run } from "@flow-state-dev/workspace"; import { readRunChanges, runChangesSchema } from "./commits"; import { z } from "zod"; @@ -355,7 +355,7 @@ export function harnessSessionFeeds( onSteerable: (steer, ctx) => steering.onSteerable(steer, ctx), // The person running the turn is the principal, so the window lands in their row. onUsage: async (report, ctx) => { - await recordHarnessUsage(ctx as never, report); + await recordHarnessUsage(ctx, report); }, model: (ctx) => (ctx.session.state as HarnessSessionState).model ?? defaultModel, cwd: async (ctx) => { @@ -440,7 +440,7 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { failure: harnessFailureKind(error), stopped: ctx.signal.aborted, error: error.message, - ...turnEndOf(ctx, requestId), + ...turnEndOnce(ctx, requestId, record), })), ); throw error; @@ -535,7 +535,7 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { execute: async (_handle: HarnessRunHandle, ctx): Promise => { const requestId = ctx.request.identity.id; try { - await changeState(ctx, (state) => withTurn(state, requestId, (record) => ({ ...record, ...turnEndOf(ctx, requestId) }))); + await changeState(ctx, (state) => withTurn(state, requestId, (record) => ({ ...record, ...turnEndOnce(ctx, requestId, record) }))); } catch (error) { ctx.emit.status(`What the turn spent was not recorded: ${(error as Error).message}`, { transient: false }); } @@ -624,28 +624,18 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { })(); } -/** What a turn's record gets when its harness step ends: what it spent, and when. */ -function turnEndOf(ctx: object, requestId: string): { spend: ReturnType; endedAt: number } { - return { spend: turnSpendOf(readRequestSpend(ctx), requestId), endedAt: Date.now() }; -} - /** - * A turn's spend, as a task's spend reads: its request's spend folded by - * `taskSpendView`, or `null` when it spent nothing (or the context carries no - * spend ledger). + * What a turn's record gets when its harness step ends: what it spent, and + * when. Once: a record that already ended keeps its end, so a second writer + * (a failure record, then the step after it) never moves it. */ -function turnSpendOf(spend: ReturnType, requestId: string) { - if (spend === undefined) return null; - const sources = requestSpendSources(spend) as TaskSpendSource[]; - if (sources.length === 0) return null; - const run = spend.harnessRuns.at(-1); - const model = Object.keys(spend.byModel).at(-1) ?? null; - return taskSpendView({ - status: "completed", - spend: { - requests: [ - { requestId, settled: sources, live: null, harness: run?.harness ?? null, harnessModel: run?.model ?? null, model, writer: null, admittedAt: null, recovered: null }, - ], - }, - }); +function turnEndOnce( + ctx: object, + requestId: string, + record: { spend?: TaskSpendView | null; endedAt?: number | null }, +): { spend: TaskSpendView | null; endedAt: number } { + if (record.endedAt != null) return { spend: record.spend ?? null, endedAt: record.endedAt }; + const spend = readRequestSpend(ctx); + return { spend: spend === undefined ? null : requestSpendView(spend, requestId), endedAt: Date.now() }; } + diff --git a/packages/orchestration/src/tasks/index.ts b/packages/orchestration/src/tasks/index.ts index 2dfb761068..06e9f6a255 100644 --- a/packages/orchestration/src/tasks/index.ts +++ b/packages/orchestration/src/tasks/index.ts @@ -31,7 +31,7 @@ export { type TaskSpendSource, type TaskSpendView, } from "./schema/spend"; -export { spendAfterWrite, taskSpendView, type TaskSpendWrite } from "./spend"; +export { requestSpendView, spendAfterWrite, taskSpendView, type TaskSpendWrite } from "./spend"; export { taskStatusSchema, isTerminalStatus, diff --git a/packages/orchestration/src/tasks/spend.ts b/packages/orchestration/src/tasks/spend.ts index a572371ac5..83705f52d7 100644 --- a/packages/orchestration/src/tasks/spend.ts +++ b/packages/orchestration/src/tasks/spend.ts @@ -138,6 +138,30 @@ export function taskSpendView(task: Pick & Partial { expect(mixed?.costUsd).toBeCloseTo(0.75); expect(mixed?.inputTokens).toBe(300); }); + + // A harness session's turn has a request and no row; it must read exactly as + // a task that ran the same request reads, or the one spend line disagrees. + it("reads one request's spend as the row of a task that ran it and settled", () => { + const spend: RequestSpend = { + byModel: {}, + harnessRuns: [ + { runId: "a", harness: "codex", model: "gpt-5-codex", inputTokens: 40, outputTokens: 4, costUsd: 0.01, billing: "api-key" }, + { runId: "b", harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 100, outputTokens: 10, costUsd: null, billing: "subscription" }, + ], + }; + const row = spendAfterWrite({ spend: null }, { requestId: "q1", part: "settle", spend }); + expect(requestSpendView(spend, "q1")).toEqual(taskSpendView({ status: "completed", spend: row })); + expect(requestSpendView(spend, "q1")).toMatchObject({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 140, costUsd: 0.01 }); + expect(requestSpendView(NONE, "q1")).toBeNull(); + }); }); interface Backing { From 6855d10fc886d7fb5bf51e7fe9ed994e0bf60c29 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 00:02:45 +0000 Subject: [PATCH 12/24] FIX-2398: a continued request's recovered charge counts once Codex on #4423: unrecovered() subtracted the recovered tokens and runs but kept the source's full costUsd, so the continuation's report of the logged run's charge counted twice. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- packages/orchestration/src/tasks/spend.ts | 2 ++ .../test/collection/record-spend.test.ts | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/packages/orchestration/src/tasks/spend.ts b/packages/orchestration/src/tasks/spend.ts index 83705f52d7..d6a9566ce8 100644 --- a/packages/orchestration/src/tasks/spend.ts +++ b/packages/orchestration/src/tasks/spend.ts @@ -180,6 +180,8 @@ function unrecovered(live: TaskSpendSource[], recovered: TaskSpendSource[] | nul outputTokens: left(source.outputTokens, logged.outputTokens), runs: left(source.runs, logged.runs), unreported: left(source.unreported, logged.unreported), + // The continuation reports the logged charge itself, as it does the tokens. + costUsd: source.costUsd === null ? null : left(source.costUsd, logged.costUsd ?? 0), }; return kept.inputTokens === 0 && kept.outputTokens === 0 && (source.kind === "model" || kept.runs === 0) ? [] : [kept]; }); diff --git a/packages/orchestration/test/collection/record-spend.test.ts b/packages/orchestration/test/collection/record-spend.test.ts index c02fda7c6a..b58617a775 100644 --- a/packages/orchestration/test/collection/record-spend.test.ts +++ b/packages/orchestration/test/collection/record-spend.test.ts @@ -180,6 +180,17 @@ describe.each([ expect(view(collection)).toMatchObject({ inputTokens: 400, outputTokens: 70, live: true }); }); + it("counts a dead run's charge once when its request is continued, as its tokens are", async () => { + // Two paid runs, one finished and logged: the continuation reports that one's charge again, so the row keeps only the other's. + const { collection, claim } = await claimed(); + const paid = (runId: string): RequestSpend["harnessRuns"][number] => ({ runId, harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 100, outputTokens: 10, costUsd: 0.1, billing: "api-key" }); + const logged: RequestSpend = { byModel: {}, harnessRuns: [paid("logged")] }; + await collection.recordSpend!("t", { requestId: "q1", part: "live", spend: { byModel: {}, harnessRuns: [paid("logged"), paid("unfinished")] }, writer: "first", admittedAt: 1 }, { claim }); + await collection.recordSpend!("t", { requestId: "q1", part: "live", spend: logged, writer: "second", admittedAt: 2, recovered: logged }, { claim }); + expect(view(collection)).toMatchObject({ inputTokens: 200, outputTokens: 20 }); + expect(view(collection)?.costUsd).toBeCloseTo(0.2); + }); + it("keeps a dead run's harness tokens when its request is continued in a new process", async () => { // A harness run whose block never finished isn't in the log, so the continuation doesn't recover it: the row is its only record. const { collection, claim } = await claimed(); From 2fac9196b5b97a0e2dcfd752d5e3d8d1c350a1f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 00:04:42 +0000 Subject: [PATCH 13/24] FIX-2398: requestSpendView is spendAfterWrite's settled row, folded Cursor on #4423: the hand-built entry duplicated spendAfterWrite's; a request's view is now the settle write and fold a task's row gets. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- packages/orchestration/src/tasks/spend.ts | 26 +++++-------------- .../test/collection/record-spend.test.ts | 5 +--- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/packages/orchestration/src/tasks/spend.ts b/packages/orchestration/src/tasks/spend.ts index d6a9566ce8..b9e7563e9a 100644 --- a/packages/orchestration/src/tasks/spend.ts +++ b/packages/orchestration/src/tasks/spend.ts @@ -139,27 +139,15 @@ export function taskSpendView(task: Pick & Partial { // A harness session's turn has a request and no row; it must read exactly as // a task that ran the same request reads, or the one spend line disagrees. - it("reads one request's spend as the row of a task that ran it and settled", () => { + it("reads one request's spend as a settled task reads: the last run named, every token and charge summed", () => { const spend: RequestSpend = { byModel: {}, harnessRuns: [ @@ -68,8 +67,6 @@ describe("taskSpendView", () => { { runId: "b", harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 100, outputTokens: 10, costUsd: null, billing: "subscription" }, ], }; - const row = spendAfterWrite({ spend: null }, { requestId: "q1", part: "settle", spend }); - expect(requestSpendView(spend, "q1")).toEqual(taskSpendView({ status: "completed", spend: row })); expect(requestSpendView(spend, "q1")).toMatchObject({ harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 140, costUsd: 0.01 }); expect(requestSpendView(NONE, "q1")).toBeNull(); }); From d3c8b22f2a0b989f4395437fe3a59fe4a818f130 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 00:14:15 +0000 Subject: [PATCH 14/24] FIX-2398: a turn continued after an interruption counts every run Codex on #4423: the once-only guard kept the interrupted run's end and dropped the continuation's spend, and a continued request's ledger holds only its own runs. The turn record now keeps the runs it counts (spentRuns), each once by runId, and every write takes the latest end. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- apps/kitchen-sink/test/harness-turn.test.ts | 2 +- packages/harness-manager/src/session-flow.ts | 52 +++++++++++++++---- .../harness-manager/test/turn-spend.spec.ts | 36 ++++++++++++- 3 files changed, 77 insertions(+), 13 deletions(-) diff --git a/apps/kitchen-sink/test/harness-turn.test.ts b/apps/kitchen-sink/test/harness-turn.test.ts index 6143595f53..01502d37ab 100644 --- a/apps/kitchen-sink/test/harness-turn.test.ts +++ b/apps/kitchen-sink/test/harness-turn.test.ts @@ -26,7 +26,7 @@ import { type PlanPanelInput, } from "@/lib/harness-turn"; -const blank: HarnessTurnRecord = { asked: null, confirmed: null, failure: null, stopped: null, error: null, at: 1, steered: null, base: null, head: null, checks: null, spend: null, endedAt: null }; +const blank: HarnessTurnRecord = { asked: null, confirmed: null, failure: null, stopped: null, error: null, at: 1, steered: null, base: null, head: null, checks: null, spend: null, endedAt: null, spentRuns: null }; describe("turnNotice", () => { it("says nothing while the turn runs", () => { diff --git a/packages/harness-manager/src/session-flow.ts b/packages/harness-manager/src/session-flow.ts index 0960ae35a0..f6a12802d9 100644 --- a/packages/harness-manager/src/session-flow.ts +++ b/packages/harness-manager/src/session-flow.ts @@ -77,7 +77,7 @@ import type { HarnessUsageHook, HarnessUsageReader, } from "@flow-state-dev/core/types"; -import { readRequestSpend } from "@flow-state-dev/core/types"; +import { readRequestSpend, type HarnessRunSpend } from "@flow-state-dev/core/types"; import { recordBranchPullRequests, run } from "@flow-state-dev/workspace"; import { readRunChanges, runChangesSchema } from "./commits"; import { z } from "zod"; @@ -124,6 +124,25 @@ export const harnessTurnRecordSchema = z.object({ spend: taskSpendViewSchema.nullable().default(null), /** When the turn ended, in epoch millis: its duration is `endedAt - at`. */ endedAt: z.number().nullable().default(null), + /** + * The harness runs `spend` counts, as each last reported. Kept so a turn + * continued in a later run of its request (after an interruption) counts + * the runs before it too: a continued request's ledger holds only its own. + */ + spentRuns: z + .array( + z.object({ + runId: z.string().optional(), + harness: z.string(), + model: z.string().nullable().optional(), + inputTokens: z.number().nullable(), + outputTokens: z.number().nullable(), + costUsd: z.number().nullable(), + billing: z.enum(["api-key", "subscription"]).nullable(), + }), + ) + .nullable() + .default(null), }); /** One turn's record. */ @@ -278,6 +297,7 @@ const BLANK_TURN: HarnessTurnRecord = { checks: null, spend: null, endedAt: null, + spentRuns: null, }; /** `state` with the turn `requestId` changed by `change`, keeping only the latest {@link KEPT_TURNS}. */ @@ -440,7 +460,7 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { failure: harnessFailureKind(error), stopped: ctx.signal.aborted, error: error.message, - ...turnEndOnce(ctx, requestId, record), + ...turnEndOf(ctx, requestId, record), })), ); // A stopped or failed turn still records where its changes end. Never in @@ -540,7 +560,7 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { execute: async (_handle: HarnessRunHandle, ctx): Promise => { const requestId = ctx.request.identity.id; try { - await changeState(ctx, (state) => withTurn(state, requestId, (record) => ({ ...record, ...turnEndOnce(ctx, requestId, record) }))); + await changeState(ctx, (state) => withTurn(state, requestId, (record) => ({ ...record, ...turnEndOf(ctx, requestId, record) }))); } catch (error) { ctx.emit.status(`What the turn spent was not recorded: ${(error as Error).message}`, { transient: false }); } @@ -631,16 +651,26 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { /** * What a turn's record gets when its harness step ends: what it spent, and - * when. Once: a record that already ended keeps its end, so a second writer - * (a failure record, then the step after it) never moves it. + * when. The runs are this run of the request's, as its ledger reports them, + * plus any an earlier run of the same request recorded that this one's + * ledger doesn't hold (a turn continued after an interruption: its ledger + * holds only its own runs). A run is one run however often it is written, so + * a second write in the same run of the request counts nothing twice. Model + * calls are the ledger's: a continued request's ledger carries the earlier + * ones. */ -function turnEndOnce( +function turnEndOf( ctx: object, requestId: string, - record: { spend?: TaskSpendView | null; endedAt?: number | null }, -): { spend: TaskSpendView | null; endedAt: number } { - if (record.endedAt != null) return { spend: record.spend ?? null, endedAt: record.endedAt }; - const spend = readRequestSpend(ctx); - return { spend: spend === undefined ? null : requestSpendView(spend, requestId), endedAt: Date.now() }; + record: { spentRuns?: readonly HarnessRunSpend[] | null }, +): { spend: TaskSpendView | null; spentRuns: HarnessRunSpend[] | null; endedAt: number } { + const pending = readRequestSpend(ctx); + const now = pending?.harnessRuns ?? []; + const sameRun = (a: HarnessRunSpend, b: HarnessRunSpend) => + a.runId !== undefined ? a.runId === b.runId : b.runId === undefined && JSON.stringify(a) === JSON.stringify(b); + const earlier = (record.spentRuns ?? []).filter((run) => !now.some((other) => sameRun(run, other))); + const harnessRuns = [...earlier, ...now]; + const spend = pending === undefined && harnessRuns.length === 0 ? null : requestSpendView({ byModel: pending?.byModel ?? {}, harnessRuns }, requestId); + return { spend, spentRuns: harnessRuns.length === 0 ? null : harnessRuns, endedAt: Date.now() }; } diff --git a/packages/harness-manager/test/turn-spend.spec.ts b/packages/harness-manager/test/turn-spend.spec.ts index 24b07a81de..744774c259 100644 --- a/packages/harness-manager/test/turn-spend.spec.ts +++ b/packages/harness-manager/test/turn-spend.spec.ts @@ -13,6 +13,8 @@ * - drop `onUsage` from the feeds: the usage query lists no window. * - write the spend from the flow's `onFinished` instead: the request's * terminal status is published before the spend lands. + * - keep a record's first end (skip a record that already ended), or count + * only the ledger's runs: a continued turn reads 10 tokens, not 20. */ import { DEFAULT_ORG_ID, handler } from "@flow-state-dev/core"; import { reportHarnessSpend, type HarnessBlock, type HarnessCallbackContext } from "@flow-state-dev/core/types"; @@ -32,6 +34,10 @@ interface Script { fail?: boolean; /** A window heard during the run. */ window?: boolean; + /** An earlier run of this request already ended the turn with this run, as a turn continued after an interruption finds it. */ + earlier?: { runId: string; inputTokens: number; outputTokens: number; costUsd: number | null }; + /** The run writes its report twice, as a harness reports its running total. */ + reportTwice?: boolean; } const stubAgent = (script: Script) => (feeds: HarnessSessionFeeds) => @@ -41,6 +47,16 @@ const stubAgent = (script: Script) => (feeds: HarnessSessionFeeds) => execute: async (_input, ctx) => { const harnessCtx = ctx as unknown as HarnessCallbackContext; await feeds.resume(harnessCtx); + const earlier = script.earlier; + if (earlier !== undefined) { + const requestId = harnessCtx.request.identity.id; + await harnessCtx.session.atomicState((state) => { + const s = state as HarnessSessionState; + const turn = s.turns?.[requestId]; + const run = { ...earlier, harness: "stub", model: "stub-model", billing: "api-key" as const }; + return { ...s, turns: { ...s.turns, [requestId]: { ...turn!, endedAt: 1, spentRuns: [run] } } }; + }); + } if (script.window === true) { await feeds.onUsage( { harness: "stub", window: { id: "five_hour", label: "5h", minutes: 300, usedPercent: 42, level: "ok", resetsAt: null }, observedAt: Date.now(), runStartedAt: Date.now() - 10, accountId: "acct" }, @@ -51,7 +67,10 @@ const stubAgent = (script: Script) => (feeds: HarnessSessionFeeds) => if (script.fail === true) throw new Error("the run broke"); return { outcome: "completed" }; } finally { - if (script.spend !== undefined) reportHarnessSpend(ctx, { runId: "run-1", harness: "stub", model: "stub-model", ...script.spend }); + if (script.spend !== undefined) { + if (script.reportTwice === true) reportHarnessSpend(ctx, { runId: "run-1", harness: "stub", model: "stub-model", ...script.spend, inputTokens: 1, outputTokens: 1 }); + reportHarnessSpend(ctx, { runId: "run-1", harness: "stub", model: "stub-model", ...script.spend }); + } } }, }) as unknown as HarnessBlock; @@ -117,6 +136,21 @@ describe("a harness session turn's spend", () => { expect(turn.spend).toBeNull(); expect(turn.endedAt).not.toBeNull(); }); + // A turn continued after an interruption runs again under the same request, + // whose ledger then holds only the new run: the turn still counts both. + it("counts the runs of an earlier run of its request too, once each", async () => { + const stores = createInMemoryStores(); + await run(stores, "s-continued", { + earlier: { runId: "run-0", inputTokens: 10, outputTokens: 1, costUsd: 0.1 }, + reportTwice: true, + spend: { inputTokens: 10, outputTokens: 1, costUsd: 0.1, billing: "api-key" }, + }); + const turn = await onlyTurn(stores, "s-continued"); + expect(turn.spend).toMatchObject({ inputTokens: 20, outputTokens: 2 }); + expect(turn.spend?.costUsd).toBeCloseTo(0.2); + expect(turn.endedAt).toBeGreaterThan(1); + }); + // A page refreshes the session when it sees the turn end, so the spend must // be stored by the time the request publishes its terminal status. it("is stored before the request publishes that it completed", async () => { From c28a08060c3f40aed3ec5e311df9ae13c047771c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 00:15:17 +0000 Subject: [PATCH 15/24] FIX-2398: the changeset names its issue, and requestSpendView Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- .changeset/turn-and-task-cost.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/turn-and-task-cost.md b/.changeset/turn-and-task-cost.md index 1e719297a3..758e2c2036 100644 --- a/.changeset/turn-and-task-cost.md +++ b/.changeset/turn-and-task-cost.md @@ -3,4 +3,4 @@ "@flow-state-dev/harness-manager": minor --- -`taskSpendView` gains `costUsd`: the sum of the charges vendors reported for the task, or `null` when none did (a subscription run reports none). `harnessSessionFlow` records what each turn spent on its turn record (`spend`, in `taskSpendView`'s shape, and `endedAt`), hands harnesses an `onUsage` feed that records subscription windows in the user's usage record, and carries a `harnessUsage` action that reads it, with optional `usageReaders`. +`taskSpendView` gains `costUsd`: the sum of the charges vendors reported for the task, or `null` when none did (a subscription run reports none). `requestSpendView` reads one request's spend the same way. `harnessSessionFlow` records what each turn spent on its turn record (`spend`, in `taskSpendView`'s shape, and `endedAt`), hands harnesses an `onUsage` feed that records subscription windows in the user's usage record, and carries a `harnessUsage` action that reads it, with optional `usageReaders` (FIX-2398). From 60ef27a7cd2bf5e31cca63f910d83fc218e86b53 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 00:18:16 +0000 Subject: [PATCH 16/24] Merge claude/project-thread-2ksmgs (fe9f981) into FIX-2398; Fallow ignores Kitchen Sink's spend copy The copy is installed for the registry drift test; the Kitchen Sink adoption PR imports it and drops the entry. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- .fallowrc.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.fallowrc.json b/.fallowrc.json index e44394b701..688defddc9 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -101,7 +101,10 @@ "apps/kitchen-sink/components/flow-state/run-changes.tsx", // Kitchen Sink's copy of the registry's run-checks, which the registry // drift test requires installed; its turn notice imports it later. - "apps/kitchen-sink/components/flow-state/run-checks.tsx" + "apps/kitchen-sink/components/flow-state/run-checks.tsx", + // Kitchen Sink's copy of the registry's spend, which the registry drift + // test requires installed; its harness session imports it later. + "apps/kitchen-sink/components/flow-state/spend.tsx" ], // Dead code fails `fallow audit` at its default severity. Complexity reports From 5378fad9cf3c315c60142b04db1a65c96cbb165f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 00:20:56 +0000 Subject: [PATCH 17/24] FIX-2398: a continued turn keeps an earlier run that has no id Codex on #4423: a report with no runId is a settled run of its own, so matching totals are not the same run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- packages/harness-manager/src/session-flow.ts | 11 ++++++----- .../harness-manager/test/turn-spend.spec.ts | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/packages/harness-manager/src/session-flow.ts b/packages/harness-manager/src/session-flow.ts index ac6214bd10..a2fe87e04e 100644 --- a/packages/harness-manager/src/session-flow.ts +++ b/packages/harness-manager/src/session-flow.ts @@ -656,8 +656,8 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { * when. The runs are this run of the request's, as its ledger reports them, * plus any an earlier run of the same request recorded that this one's * ledger doesn't hold (a turn continued after an interruption: its ledger - * holds only its own runs). A run is one run however often it is written, so - * a second write in the same run of the request counts nothing twice. Model + * holds only its own runs). A run with an id is one run however often it is + * written, so a second write in the same run of the request counts it once. Model * calls are the ledger's: a continued request's ledger carries the earlier * ones. */ @@ -668,9 +668,10 @@ function turnEndOf( ): { spend: TaskSpendView | null; spentRuns: HarnessRunSpend[] | null; endedAt: number } { const pending = readRequestSpend(ctx); const now = pending?.harnessRuns ?? []; - const sameRun = (a: HarnessRunSpend, b: HarnessRunSpend) => - a.runId !== undefined ? a.runId === b.runId : b.runId === undefined && JSON.stringify(a) === JSON.stringify(b); - const earlier = (record.spentRuns ?? []).filter((run) => !now.some((other) => sameRun(run, other))); + // A run with an id is the same run wherever it is written. One without is a + // settled run of its own (the `HarnessRunSpend` contract), so an earlier + // one is always kept: the turn writes once per run of its request. + const earlier = (record.spentRuns ?? []).filter((run) => run.runId === undefined || !now.some((other) => other.runId === run.runId)); const harnessRuns = [...earlier, ...now]; const spend = pending === undefined && harnessRuns.length === 0 ? null : requestSpendView({ byModel: pending?.byModel ?? {}, harnessRuns }, requestId); return { spend, spentRuns: harnessRuns.length === 0 ? null : harnessRuns, endedAt: Date.now() }; diff --git a/packages/harness-manager/test/turn-spend.spec.ts b/packages/harness-manager/test/turn-spend.spec.ts index 744774c259..8f92e69d44 100644 --- a/packages/harness-manager/test/turn-spend.spec.ts +++ b/packages/harness-manager/test/turn-spend.spec.ts @@ -35,7 +35,9 @@ interface Script { /** A window heard during the run. */ window?: boolean; /** An earlier run of this request already ended the turn with this run, as a turn continued after an interruption finds it. */ - earlier?: { runId: string; inputTokens: number; outputTokens: number; costUsd: number | null }; + earlier?: { runId?: string; inputTokens: number; outputTokens: number; costUsd: number | null }; + /** The run reports with no id, as a harness that settles each report does. */ + anonymous?: boolean; /** The run writes its report twice, as a harness reports its running total. */ reportTwice?: boolean; } @@ -53,7 +55,7 @@ const stubAgent = (script: Script) => (feeds: HarnessSessionFeeds) => await harnessCtx.session.atomicState((state) => { const s = state as HarnessSessionState; const turn = s.turns?.[requestId]; - const run = { ...earlier, harness: "stub", model: "stub-model", billing: "api-key" as const }; + const run = { harness: "stub", model: "stub-model", ...earlier, billing: "api-key" as const }; return { ...s, turns: { ...s.turns, [requestId]: { ...turn!, endedAt: 1, spentRuns: [run] } } }; }); } @@ -69,7 +71,7 @@ const stubAgent = (script: Script) => (feeds: HarnessSessionFeeds) => } finally { if (script.spend !== undefined) { if (script.reportTwice === true) reportHarnessSpend(ctx, { runId: "run-1", harness: "stub", model: "stub-model", ...script.spend, inputTokens: 1, outputTokens: 1 }); - reportHarnessSpend(ctx, { runId: "run-1", harness: "stub", model: "stub-model", ...script.spend }); + reportHarnessSpend(ctx, { ...(script.anonymous === true ? {} : { runId: "run-1" }), harness: "stub", model: "stub-model", ...script.spend }); } } }, @@ -151,6 +153,17 @@ describe("a harness session turn's spend", () => { expect(turn.endedAt).toBeGreaterThan(1); }); + it("keeps an earlier run with no id even when its totals match this one's", async () => { + const stores = createInMemoryStores(); + await run(stores, "s-anonymous", { + earlier: { inputTokens: 10, outputTokens: 1, costUsd: 0.1 }, + anonymous: true, + spend: { inputTokens: 10, outputTokens: 1, costUsd: 0.1, billing: "api-key" }, + }); + const turn = await onlyTurn(stores, "s-anonymous"); + expect(turn.spend).toMatchObject({ inputTokens: 20, outputTokens: 2 }); + }); + // A page refreshes the session when it sees the turn end, so the spend must // be stored by the time the request publishes its terminal status. it("is stored before the request publishes that it completed", async () => { From 7d92efe51bf92b3fc7b59bbc005627459efe51ed Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 00:33:53 +0000 Subject: [PATCH 18/24] FIX-2398: the engine names a harness run reported without an id A turn that wrote its spend and then lost its process before its step's trace was stored replays that write on continuation, while the ledger recovers the same id-less run from the harness step's trace. The turn record kept every id-less earlier run, so that run counted twice. The engine now gives an id-less report a settled_ id as it counts it, so the ledger, the trace and what a continuation recovers all name the run the same, and the turn dedupes it by id like any other run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- .changeset/settled-run-ids.md | 6 ++++++ packages/core/README.md | 2 +- packages/core/src/types/harness.ts | 7 +++++-- packages/core/src/types/request-end.ts | 3 ++- .../src/context/createExecutionContext.ts | 9 ++++++-- packages/engine/test/request-spend.test.ts | 21 +++++++++++++------ packages/harness-manager/src/session-flow.ts | 8 ++++--- 7 files changed, 41 insertions(+), 15 deletions(-) create mode 100644 .changeset/settled-run-ids.md diff --git a/.changeset/settled-run-ids.md b/.changeset/settled-run-ids.md new file mode 100644 index 0000000000..04c668fe41 --- /dev/null +++ b/.changeset/settled-run-ids.md @@ -0,0 +1,6 @@ +--- +"@flow-state-dev/engine": patch +"@flow-state-dev/core": patch +--- + +A harness run reported without a `runId` is given one (`settled_…`) as the request counts it. Request-end hooks, `readRequestSpend` and a continuation that recovers the run from the log all see the same id, so a reader that wrote the run down can tell it from a second run (FIX-2398). diff --git a/packages/core/README.md b/packages/core/README.md index 7ddadc267b..79ac64ce58 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -465,7 +465,7 @@ Set `client: { live: true }` to stream each mutation's projected `clientData` as - The plan record (a harness conversation's to-do list), from the package root: `OBSERVED_PLAN`, `observedPlanCollection`, `observedPlanItemStateSchema`, the type `ObservedOutcome` (with `observedOutcomeSchema`), the list markers (`OBSERVED_PLAN_LIST`, `observedPlanListCollection`) and `replacePlanList(plan, marker, namespace, items, { now?, requestId? })`, which writes a whole-list plan as entries and stamps `requestId` on each. One shape for a coding agent's own to-do list, which a harness that reports one declares and writes. A record of what the agent planned, never a work queue. A plan belongs to the harness conversation: `harnessConversationNamespace(conversationId)` returns the namespace it is keyed under (`conversation/`), and `planConversationPrefix(conversationId)` the key prefix of its entries. Each run leaves a pointer marker under its own namespace (`harnessRunNamespace(ctx)`) whose `conversation` field names that namespace. To read one request's plan, list its markers with `planRowsPrefix(requestId, OBSERVED_PLAN_LIST)` as `topicPrefix` and pass them to `readPlanTarget(requestId, markers)` (markers typed `PlanMarkerRead`, `{ topic, conversation? }`): it returns `{ kind: "conversation", namespace, rowsPrefix }` when a run left a pointer, and `{ kind: "run", rowsPrefix }` for a request whose plan is stored under its own runs. `planKeySegment(value)` is how an id or run name is written as one key segment. The keys, collections, `harnessConversationNamespace`, `planConversationPrefix`, `planRowsPrefix`, `planKeySegment` and `readPlanTarget` are also exported from `@flow-state-dev/core/types`, for browser code. Guide: [the run's plan](https://flow-state.dev/docs/tools/coding-agents#the-runs-plan). - The pull-request record (the pull requests a session opened), from the package root: `OBSERVED_PULL_REQUESTS`, `observedPullRequestsCollection` (session-scoped, live, readable from the browser), `observedPullRequestStateSchema` (type `ObservedPullRequestState`: `host`, `repository` as `owner/name`, `number`, `url`, `title`, `state`, `branch`, `requestId`, `source` (`"tool"` or `"branch"`) and `firstSeenAt`), `pullRequestKey(ref)`, `pullRequestRefFromUrl(url)` (a pull request's link read as a ref, the one reading every writer keys by; type `PullRequestLink`) and the type `PullRequestRef`. One row per pull request, keyed by host, repository and number, so a pull request seen twice is one row. `writePullRequests(collection, refs, { source, requestId?, now? })` is the one writer: it fills only fields that are still empty, never moves `requestId` or `firstSeenAt`, and a `"tool"` sighting upgrades a `"branch"` row. `requestId` is the request that opened the pull request, so a reader narrows a session's rows to one run by it. What counts as opened is a rule: `githubPullRequestRule` (in `DEFAULT_PULL_REQUEST_RULES`) reads a `gh pr create` command or a `create_pull_request` tool whose output holds the pull request's link. A `PullRequestRule` is `{ name, match(call) }`; `findOpenedPullRequests(item, rules?)` applies rules to one `tool_output` item. `recordPullRequests(ctx, { rules? })` starts the request's recorder, which writes each pull request as its call settles and stops when the request ends; the Codex, Claude Code and Cursor blocks call it. `pullRequestWritesSettled(response)` resolves once the request's recorder has finished the writes it started, for a writer that runs at request end. `pullRequestRecorder({ rules?, name? })` is the same recorder as a handler block, for a flow with no harness: add it as a `.tap` step. With no collection declared, it records nothing and says so in a status note. The key, collection, schema and `pullRequestKey` are also exported from `@flow-state-dev/core/types`. Guide: [pull requests a session opened](https://flow-state.dev/docs/tools/coding-agents#pull-requests-a-session-opened). - Subscription usage: `harnessUsageWindowSchema` from the package root, and the types `HarnessUsageWindow`, `HarnessUsageLevel`, `HarnessUsageReading`, `HarnessUsageReader`, `HarnessUsageReport` and `HarnessUsageHook` from `@flow-state-dev/core/types`. A reader `(options?: { signal }) => Promise` asks a harness how much of each subscription window is used, with no run. A reading is `windows` (each window's `id`, `label`, `minutes`, `usedPercent` 0 to 100 or `null` when the vendor sent none, `level` (`"ok"`, `"warning"`, `"reached"` or `null`) and `resetsAt` in epoch ms, plus `plan` and `accountId`) or one of `api-key`, `signed-out` and `unavailable`, each with a `reason` and no number. A reader returns those states rather than throwing. A harness that reports usage during a run calls a `HarnessUsageHook` `(report, ctx)` with one `HarnessUsageReport` per window: `{ harness, window, observedAt, runStartedAt, accountId }`. The schema reads a window with no `level` as `level: null`. Guide: [subscription usage](https://flow-state.dev/docs/tools/coding-agents#subscription-usage). -- Run spend: `reportHarnessSpend(ctx, spend)` and the types `HarnessRunSpend` and `HarnessBilling`, from `@flow-state-dev/core/types`. A harness block calls it as a run goes and once more when the run ends however it ends, with `{ harness, inputTokens, outputTokens, costUsd, billing, runId, model }`; a later report with the same `runId` replaces the earlier one, never adds to it, and `model` is the model the run named: what the agent reported, each `null` when it reported none (never 0, never an estimate), and `billing` `"api-key"`, `"subscription"` or `null`. It returns `false` and counts nothing outside a request. The run is counted in the request's spend, never in `ctx.request.tokenUsage` or an action's `tokenBudget`. Guide: [what a run spent](https://flow-state.dev/docs/tools/coding-agents#what-a-run-spent). +- Run spend: `reportHarnessSpend(ctx, spend)` and the types `HarnessRunSpend` and `HarnessBilling`, from `@flow-state-dev/core/types`. A harness block calls it as a run goes and once more when the run ends however it ends, with `{ harness, inputTokens, outputTokens, costUsd, billing, runId, model }`; a later report with the same `runId` replaces the earlier one, never adds to it (a report without one is a settled run of its own, and the request names it with a fresh `settled_…` id), and `model` is the model the run named: what the agent reported, each `null` when it reported none (never 0, never an estimate), and `billing` `"api-key"`, `"subscription"` or `null`. It returns `false` and counts nothing outside a request. The run is counted in the request's spend, never in `ctx.request.tokenUsage` or an action's `tokenBudget`. Guide: [what a run spent](https://flow-state.dev/docs/tools/coding-agents#what-a-run-spent). - `harnessFailureKind(error)`, from the package root (type `HarnessFailureKind` from `@flow-state-dev/core/types`): `"unknown-conversation"` when a harness refused to continue the conversation a run asked to `resume`, `null` for any other failure. Follows `cause`, so it reads through wrapped errors. A harness package sets it on the error it throws. - `formatRunDuration(ms)`, from the package root: a readable string for a duration in milliseconds, rounded to the nearest second. `formatRunDuration(324_000)` is `"5 mins, 24 seconds"`. A run of an hour or more shows hours and minutes only (`"1 hr, 3 mins"`). A duration under half a second, a negative number, or `NaN` gives `"less than a second"`. The Claude Code, Codex and Cursor blocks use it for their run-end status. diff --git a/packages/core/src/types/harness.ts b/packages/core/src/types/harness.ts index 115cc86931..f9f282a834 100644 --- a/packages/core/src/types/harness.ts +++ b/packages/core/src/types/harness.ts @@ -483,7 +483,9 @@ export interface HarnessRunSpend { /** * The harness's own id for this run (FIX-2288). A later report with the same * id replaces the earlier one, so a harness reports its running total, never - * an increment. Absent: the report is one settled run of its own. + * an increment. Absent: the report is one settled run of its own, and the + * engine names it (a fresh `settled_…` id) as it counts it, so what a reader + * reads back, and what a continuation recovers from the log, carries that id. */ runId?: string; /** The harness, by the name a usage record is kept under (`claude-code`, `codex`, `cursor`). */ @@ -516,7 +518,8 @@ export interface HarnessRunSpend { * tokens `null`), on each usage event its vendor sends (the run's total so * far), and from its `finally` when the run ends however it ends (the final * total). Each report carries the whole run so far, never an increment, and - * the run counts once. Without a `runId`, each call is one settled run. + * the run counts once. Without a `runId`, each call is one settled run, which + * the engine gives an id of its own. * * @returns `false` when the context carries no spend ledger: a hand-built * test context, or a block run outside a request. Nothing is counted then. diff --git a/packages/core/src/types/request-end.ts b/packages/core/src/types/request-end.ts index bc05d8f788..323948d2a8 100644 --- a/packages/core/src/types/request-end.ts +++ b/packages/core/src/types/request-end.ts @@ -174,7 +174,8 @@ export const REQUEST_SPEND_SEAM: unique symbol = Symbol.for("@flow-state-dev/req export interface RequestSpendSeam { /** * Add one harness report to the request's spend. A report with a `runId` - * replaces that run's earlier report; one without is one settled run. + * replaces that run's earlier report; one without is one settled run. The + * engine's seam names that run before it counts it. */ reportHarnessRun(run: HarnessRunSpend): void; /** diff --git a/packages/engine/src/context/createExecutionContext.ts b/packages/engine/src/context/createExecutionContext.ts index ca44b5ff4c..a4c7092bad 100644 --- a/packages/engine/src/context/createExecutionContext.ts +++ b/packages/engine/src/context/createExecutionContext.ts @@ -4191,11 +4191,16 @@ export async function createExecutionContext< // What this block's own harness runs spent also goes on its completed // trace, so a continuation after the process died can hand it on (FIX-2145). // A run that reports as it goes (FIX-2288) is listed once, at its latest report. + // A report with no run id is a settled run of its own, so it is given + // an id here: the ledger and the trace then name it alike, and a reader + // that recorded it can tell it from another run after a continuation + // recovers it from the trace (FIX-2398). const harnessSpendOfBlock: HarnessRunSpend[] = []; (childContext as { [REQUEST_SPEND_SEAM]?: RequestSpendSeam })[REQUEST_SPEND_SEAM] = { - reportHarnessRun(run) { + reportHarnessRun(report) { + const run = report.runId === undefined ? { ...report, runId: generateId("settled") } : report; requestSpendLedger.reportHarnessRun(run); - const at = run.runId === undefined ? -1 : harnessSpendOfBlock.findIndex((reported) => reported.runId === run.runId); + const at = harnessSpendOfBlock.findIndex((reported) => reported.runId === run.runId); if (at >= 0) harnessSpendOfBlock.splice(at, 1); harnessSpendOfBlock.push({ ...run }); }, diff --git a/packages/engine/test/request-spend.test.ts b/packages/engine/test/request-spend.test.ts index 73860ac0a3..431b02a800 100644 --- a/packages/engine/test/request-spend.test.ts +++ b/packages/engine/test/request-spend.test.ts @@ -140,7 +140,8 @@ describe("request spend ledger", () => { expect(endings).toHaveLength(1); expect(endings[0]!.spend).toEqual({ byModel: { "openai/gpt-6-luna": { prompt: 8, completion: 8, total: 16, cacheReadTokens: 0, cacheCreationTokens: 0 } }, - harnessRuns: [{ harness: "claude-code", inputTokens: 1000, outputTokens: 200, costUsd: 0.42, billing: "api-key" }] + // A report with no run id is named here, so whoever records it can tell it from another run. + harnessRuns: [{ runId: expect.stringMatching(/^settled_/), harness: "claude-code", inputTokens: 1000, outputTokens: 200, costUsd: 0.42, billing: "api-key" }] }); }); @@ -687,12 +688,14 @@ describe("request spend ledger", () => { process.env.FSDEV_TRACE_OBSERVABILITY = observability; const endings: RequestEnding[] = []; let harnessRuns = 0; + let liveRunId: string | undefined; const harnessStep = handler({ name: "harness-step", inputSchema: z.any(), execute: (value, ctx) => { harnessRuns += 1; reportHarnessSpend(ctx, { harness: "codex", inputTokens: 100, outputTokens: 20, costUsd: 0.5, billing: "api-key" }); + liveRunId = readRequestSpend(ctx)?.harnessRuns[0]?.runId; return value; } }); @@ -735,8 +738,11 @@ describe("request spend ledger", () => { await deadRun.catch(() => undefined); expect(harnessRuns).toBe(1); + // The run the continuation recovers is the one the dead run read, by + // name: whoever wrote it down then can tell it is not a second run. + expect(liveRunId).toMatch(/^settled_/); expect(endingsBeforeTeardown).toEqual([ - ["completed", [{ harness: "codex", inputTokens: 100, outputTokens: 20, costUsd: 0.5, billing: "api-key" }]] + ["completed", [{ runId: liveRunId, harness: "codex", inputTokens: 100, outputTokens: 20, costUsd: 0.5, billing: "api-key" }]] ]); } ); @@ -955,13 +961,16 @@ describe("request spend read mid-run", () => { await run(flow, {}, "runs"); - // Run "a" was reported last, so it is last; the two id-less reports are two runs. - expect(endings[0]!.spend.harnessRuns.map((r) => [r.runId ?? "-", r.harness, r.inputTokens])).toEqual([ + // Run "a" was reported last, so it is last; the two id-less reports are + // two runs, each named apart. + const runs = endings[0]!.spend.harnessRuns; + expect(runs.map((r) => [r.runId?.startsWith("settled_") ? "settled" : r.runId, r.harness, r.inputTokens])).toEqual([ ["b", "codex", 2], - ["-", "cursor", 5], - ["-", "cursor", 5], + ["settled", "cursor", 5], + ["settled", "cursor", 5], ["a", "codex", 3] ]); + expect(runs[1]!.runId).not.toBe(runs[2]!.runId); }); it("keys model calls in the order each model was last called, so a task names the model it called last", () => { diff --git a/packages/harness-manager/src/session-flow.ts b/packages/harness-manager/src/session-flow.ts index a2fe87e04e..206b9e5458 100644 --- a/packages/harness-manager/src/session-flow.ts +++ b/packages/harness-manager/src/session-flow.ts @@ -668,9 +668,11 @@ function turnEndOf( ): { spend: TaskSpendView | null; spentRuns: HarnessRunSpend[] | null; endedAt: number } { const pending = readRequestSpend(ctx); const now = pending?.harnessRuns ?? []; - // A run with an id is the same run wherever it is written. One without is a - // settled run of its own (the `HarnessRunSpend` contract), so an earlier - // one is always kept: the turn writes once per run of its request. + // A run with an id is the same run wherever it is written, including one a + // continuation recovered from the log after this turn's earlier write (the + // engine names a run reported without an id, so that one has one too). One + // still without an id came from a seam that named none: a settled run of its + // own (the `HarnessRunSpend` contract), so an earlier one is always kept. const earlier = (record.spentRuns ?? []).filter((run) => run.runId === undefined || !now.some((other) => other.runId === run.runId)); const harnessRuns = [...earlier, ...now]; const spend = pending === undefined && harnessRuns.length === 0 ? null : requestSpendView({ byModel: pending?.byModel ?? {}, harnessRuns }, requestId); From 7e9add20a955866fb589209a3dbec719432dc4b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 00:55:20 +0000 Subject: [PATCH 19/24] ci: pin Kitchen-sink E2E to ubuntu-24.04 (ported from FIX-2438, #4431) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fbb6dd942..e483cad51d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -440,7 +440,10 @@ jobs: e2e: name: Kitchen-sink E2E (Playwright) - runs-on: ubuntu-latest + # Pinned: the installed Playwright can't install chromium-headless-shell + # on ubuntu-26.04, which ubuntu-latest now resolves to. Unpin once + # Playwright supports 26.04 (FIX-2438). + runs-on: ubuntu-24.04 timeout-minutes: 10 # needs is queue order, not a build dependency. Still runs if Typecheck & Test fails. needs: [ci] From 1cbfb5cc3dc2af52bbb1084685dc0c2e1f5b7f1a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 01:14:28 +0000 Subject: [PATCH 20/24] FIX-2398: no tracking id in the engine's settled-run comment Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- packages/engine/src/context/createExecutionContext.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/engine/src/context/createExecutionContext.ts b/packages/engine/src/context/createExecutionContext.ts index a4c7092bad..c872bd0638 100644 --- a/packages/engine/src/context/createExecutionContext.ts +++ b/packages/engine/src/context/createExecutionContext.ts @@ -4194,7 +4194,7 @@ export async function createExecutionContext< // A report with no run id is a settled run of its own, so it is given // an id here: the ledger and the trace then name it alike, and a reader // that recorded it can tell it from another run after a continuation - // recovers it from the trace (FIX-2398). + // recovers it from the trace. const harnessSpendOfBlock: HarnessRunSpend[] = []; (childContext as { [REQUEST_SPEND_SEAM]?: RequestSpendSeam })[REQUEST_SPEND_SEAM] = { reportHarnessRun(report) { From 7ef39e45bc601234396b2aa47a903901244bf1af Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 01:30:57 +0000 Subject: [PATCH 21/24] FIX-2398: a continued turn keeps the model calls its first run made A disconnect ends the request's first run, whose ending hands its model calls on; the continuation replays the finished calls without making them again, so the pending slice no longer holds them and the turn's spend lost them. The turn now reads the request's whole model count (ctx.request.tokenUsage.byModel), which carries what earlier endings handed on and counts recovered calls once. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- packages/harness-manager/src/session-flow.ts | 14 ++--- .../harness-manager/test/turn-spend.spec.ts | 52 ++++++++++++++++++- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/packages/harness-manager/src/session-flow.ts b/packages/harness-manager/src/session-flow.ts index 9c1f8fe8ab..33441122f0 100644 --- a/packages/harness-manager/src/session-flow.ts +++ b/packages/harness-manager/src/session-flow.ts @@ -77,7 +77,7 @@ import type { HarnessUsageHook, HarnessUsageReader, } from "@flow-state-dev/core/types"; -import { readRequestSpend, type HarnessRunSpend } from "@flow-state-dev/core/types"; +import { readRequestSpend, type HarnessRunSpend, type ModelUsageEntry } from "@flow-state-dev/core/types"; import { recordBranchPullRequests, run } from "@flow-state-dev/workspace"; import { readRunChanges, runChangesSchema } from "./commits"; import { z } from "zod"; @@ -550,7 +550,7 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { }); // When the harness step returns: what the turn spent, on its record. The // harness reported its run into the request before its block returned, so - // the request's pending spend is the whole turn. Written inside the turn, + // the request's spend is the whole turn. Written inside the turn, // not at `onFinished`, so it lands before the request publishes its end and // a page that refreshes on that end reads it. A turn that threw records it // in `recordFailure`'s write instead. A failed write is a note, never a @@ -658,11 +658,12 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { * ledger doesn't hold (a turn continued after an interruption: its ledger * holds only its own runs). A run with an id is one run however often it is * written, so a second write in the same run of the request counts it once. Model - * calls are the ledger's: a continued request's ledger carries the earlier - * ones. + * calls are the request's whole count (`ctx.request.tokenUsage`), not its + * pending slice: an interruption's ending hands its calls on, and a + * continuation replays them without making them again. */ function turnEndOf( - ctx: object, + ctx: { request: { tokenUsage: { byModel: Record } } }, requestId: string, record: { spentRuns?: readonly HarnessRunSpend[] | null }, ): { spend: TaskSpendView | null; spentRuns: HarnessRunSpend[] | null; endedAt: number } { @@ -675,7 +676,8 @@ function turnEndOf( // own (the `HarnessRunSpend` contract), so an earlier one is always kept. const earlier = (record.spentRuns ?? []).filter((run) => run.runId === undefined || !now.some((other) => other.runId === run.runId)); const harnessRuns = [...earlier, ...now]; - const spend = pending === undefined && harnessRuns.length === 0 ? null : requestSpendView({ byModel: pending?.byModel ?? {}, harnessRuns }, requestId); + const byModel = ctx.request.tokenUsage.byModel; + const spend = pending === undefined && harnessRuns.length === 0 ? null : requestSpendView({ byModel, harnessRuns }, requestId); return { spend, spentRuns: harnessRuns.length === 0 ? null : harnessRuns, endedAt: Date.now() }; } diff --git a/packages/harness-manager/test/turn-spend.spec.ts b/packages/harness-manager/test/turn-spend.spec.ts index 8f92e69d44..36257f0961 100644 --- a/packages/harness-manager/test/turn-spend.spec.ts +++ b/packages/harness-manager/test/turn-spend.spec.ts @@ -15,10 +15,12 @@ * terminal status is published before the spend lands. * - keep a record's first end (skip a record that already ended), or count * only the ledger's runs: a continued turn reads 10 tokens, not 20. + * - read model calls from the request's pending slice: a turn continued + * after a disconnect loses the calls its first run made. */ -import { DEFAULT_ORG_ID, handler } from "@flow-state-dev/core"; +import { DEFAULT_ORG_ID, generator, handler, sequencer } from "@flow-state-dev/core"; import { reportHarnessSpend, type HarnessBlock, type HarnessCallbackContext } from "@flow-state-dev/core/types"; -import { createInMemoryStores, runAction } from "@flow-state-dev/engine"; +import { continueRequest, createCheckpointDurabilityProvider, createFlowRegistry, createInMemoryStores, runAction } from "@flow-state-dev/engine"; import { describe, expect, it } from "vitest"; import { z } from "zod"; @@ -164,6 +166,52 @@ describe("a harness session turn's spend", () => { expect(turn.spend).toMatchObject({ inputTokens: 20, outputTokens: 2 }); }); + // A disconnect ends the request's first run, which hands its spend on; the + // continuation replays the finished model call without making it again, so + // the turn's spend is the request's whole count, not what is left to hand on. + it("keeps the model calls a turn made before a disconnect when the turn is continued", async () => { + const stores = createInMemoryStores(); + await born(stores, "s-disconnect"); + let waited = false; + const waitOnce = handler({ + name: "wait-once", + inputSchema: z.any(), + execute: async (value, ctx) => { + if (waited) return value; + waited = true; + await new Promise((_resolve, reject) => ctx.signal.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError")))); + return value; + }, + }); + const agent = () => + sequencer({ name: "stub-harness", durable: true, inputSchema: z.object({ prompt: z.string() }) }) + .step(generator({ name: "ask", model: "openai/gpt-6-luna", prompt: () => "p", user: () => "u" })) + .step(waitOnce) + .step(handler({ name: "done", inputSchema: z.any(), execute: () => ({ outcome: "completed" }) })) as unknown as HarnessBlock; + const flow = harnessSessionFlow({ kind: "stub-session", workspace, agent }); + const modelResolver = Object.assign( + () => ({ modelId: "openai/gpt-6-luna", generate: async () => ({ text: "ok", usage: { promptTokens: 10, completionTokens: 2, totalTokens: 12 } }) }), + { resolveId: (id: string) => id }, + ); + const provider = createCheckpointDurabilityProvider({ checkpoints: stores.checkpoints, suspensions: stores.suspensions, leases: stores.leases }); + const runtimeConfig = { durabilityProvider: provider, modelResolver }; + const disconnect = new AbortController(); + + const first = runAction({ flow, actionName: "turn", input: { prompt: "hello" }, userId: "u", orgId: DEFAULT_ORG_ID, sessionId: "s-disconnect", requestId: "r-disconnect", signal: disconnect.signal, stores, runtimeConfig } as never); + while (!waited) await new Promise((resolve) => setTimeout(resolve, 5)); + disconnect.abort(); + await (first as Promise).catch(() => undefined); + expect((await stores.request.get("r-disconnect"))?.status).toBe("interrupted"); + const registry = createFlowRegistry(); + registry.register(flow as never); + const { finished } = await continueRequest({ requestId: "r-disconnect", stores, flowRegistry: registry, runtimeConfig } as never); + await finished; + + expect((await stores.request.get("r-disconnect"))?.status).toBe("completed"); + const turn = await onlyTurn(stores, "s-disconnect"); + expect(turn.spend).toMatchObject({ model: "openai/gpt-6-luna", inputTokens: 10, outputTokens: 2 }); + }); + // A page refreshes the session when it sees the turn end, so the spend must // be stored by the time the request publishes its terminal status. it("is stored before the request publishes that it completed", async () => { From 12b1a3e917a35a48992d0fe6306e4313fa570b0e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 02:00:02 +0000 Subject: [PATCH 22/24] FIX-2398: a request carries its harness runs across endings; a turn reads its total A run reported before a suspension (or an interruption) was settled by that ending and never seen again: the resumed ledger carried only model calls, and the turn's spentRuns only covered a write that the failure path made. The request record now carries spentHarnessRuns beside spentByModel, the ledger starts its total from them (never handing them on again), and core's readRequestTotalSpend reads the whole request's spend. The harness turn reads that total, so its spentRuns carry is gone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- .changeset/settled-run-ids.md | 6 +- apps/kitchen-sink/test/harness-turn.test.ts | 2 +- packages/core/README.md | 2 +- packages/core/src/types/index.ts | 1 + packages/core/src/types/request-end.ts | 24 +++++ .../src/context/createExecutionContext.ts | 7 +- .../src/execution/request-spend-ledger.ts | 13 ++- packages/engine/src/execution/runAction.ts | 12 ++- packages/engine/src/stores/types.ts | 10 +- packages/engine/test/request-spend.test.ts | 14 +++ packages/harness-manager/src/session-flow.ts | 58 ++--------- .../harness-manager/test/turn-spend.spec.ts | 99 +++++++++++-------- 12 files changed, 146 insertions(+), 102 deletions(-) diff --git a/.changeset/settled-run-ids.md b/.changeset/settled-run-ids.md index 04c668fe41..cae4d48f85 100644 --- a/.changeset/settled-run-ids.md +++ b/.changeset/settled-run-ids.md @@ -1,6 +1,6 @@ --- -"@flow-state-dev/engine": patch -"@flow-state-dev/core": patch +"@flow-state-dev/engine": minor +"@flow-state-dev/core": minor --- -A harness run reported without a `runId` is given one (`settled_…`) as the request counts it. Request-end hooks, `readRequestSpend` and a continuation that recovers the run from the log all see the same id, so a reader that wrote the run down can tell it from a second run (FIX-2398). +A harness run reported without a `runId` gets a `settled_…` id when the request counts it. Request-end hooks, `readRequestSpend`, and a continuation that recovers the run from the log all see that same id. A request's record now carries its harness runs across a suspension or an interruption (`spentHarnessRuns`), as it already carried model calls. The new `readRequestTotalSpend(ctx)` reads everything the request has spent across all its runs, so a record of the whole request, such as a harness turn, reads the same after a resume (FIX-2398). diff --git a/apps/kitchen-sink/test/harness-turn.test.ts b/apps/kitchen-sink/test/harness-turn.test.ts index 01502d37ab..6143595f53 100644 --- a/apps/kitchen-sink/test/harness-turn.test.ts +++ b/apps/kitchen-sink/test/harness-turn.test.ts @@ -26,7 +26,7 @@ import { type PlanPanelInput, } from "@/lib/harness-turn"; -const blank: HarnessTurnRecord = { asked: null, confirmed: null, failure: null, stopped: null, error: null, at: 1, steered: null, base: null, head: null, checks: null, spend: null, endedAt: null, spentRuns: null }; +const blank: HarnessTurnRecord = { asked: null, confirmed: null, failure: null, stopped: null, error: null, at: 1, steered: null, base: null, head: null, checks: null, spend: null, endedAt: null }; describe("turnNotice", () => { it("says nothing while the turn runs", () => { diff --git a/packages/core/README.md b/packages/core/README.md index 79ac64ce58..42e1cf3ebf 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -961,7 +961,7 @@ assertStrictCompatible(z.object({ pairs: z.array(z.object({ key: z.string(), val assertStrictCompatible(z.object({ metrics: z.record(z.string(), z.number()) })); ``` -**Request spend.** Every model result a generator or evaluator reports (each tool-loop step, an evaluator called as a tool, a schema-repair call) is added to `ctx.request.tokenUsage`. See [token budgets](https://flow-state.dev/docs/configuration/flow#token-budgets). `RequestSpend` (`{ byModel, harnessRuns }`, from `@flow-state-dev/core/types`) is what a request spent: its model calls by model and its harness runs. A request-end hook gets it as `RequestEnding.spend`, counted since the request's last ending, so a request that suspends and resumes hands each part on once. `SESSION_REQUEST_END_HOOK` (type `SessionRequestEndHook`) and `REQUEST_SPEND_SEAM` are framework-facing slots that framework packages fill (Workforce's `installation.session()` uses the first to record [worker usage](https://flow-state.dev/docs/workforce/worker-usage)); a flow author doesn't set them. `readRequestSpend(ctx)` reads what the running request has spent so far without taking it from its ending, and `requestSpendSources(spend)` turns a `RequestSpend` into one entry per model and per harness run (`RequestSpendSource`). +**Request spend.** Every model result a generator or evaluator reports (each tool-loop step, an evaluator called as a tool, a schema-repair call) is added to `ctx.request.tokenUsage`. See [token budgets](https://flow-state.dev/docs/configuration/flow#token-budgets). `RequestSpend` (`{ byModel, harnessRuns }`, from `@flow-state-dev/core/types`) is what a request spent: its model calls by model and its harness runs. A request-end hook gets it as `RequestEnding.spend`, counted since the request's last ending, so a request that suspends and resumes hands each part on once. `SESSION_REQUEST_END_HOOK` (type `SessionRequestEndHook`) and `REQUEST_SPEND_SEAM` are framework-facing slots that framework packages fill (Workforce's `installation.session()` uses the first to record [worker usage](https://flow-state.dev/docs/workforce/worker-usage)); a flow author doesn't set them. `readRequestSpend(ctx)` reads what the running request has spent since its last ending without taking it from that ending; `readRequestTotalSpend(ctx)` reads everything it has spent across all its runs, including what a suspension or an interruption already handed on (a record of the whole request reads this one); and `requestSpendSources(spend)` turns a `RequestSpend` into one entry per model and per harness run (`RequestSpendSource`). `generator()` calls `assertStrictCompatible` automatically at definition when an `outputSchema` is set, so a bad output schema fails at import — not on the first live model call. Call it directly only to assert a bare schema constant in a test. See BP-016 in `docs/contributing/best-practices.md`. diff --git a/packages/core/src/types/index.ts b/packages/core/src/types/index.ts index d205635855..0269f00b63 100644 --- a/packages/core/src/types/index.ts +++ b/packages/core/src/types/index.ts @@ -118,6 +118,7 @@ export { addRequestSpendSource, onRequestEnded, readRequestSpend, + readRequestTotalSpend, requestSpendSources } from "./request-end"; export type { diff --git a/packages/core/src/types/request-end.ts b/packages/core/src/types/request-end.ts index 323948d2a8..4be685a1ec 100644 --- a/packages/core/src/types/request-end.ts +++ b/packages/core/src/types/request-end.ts @@ -184,6 +184,13 @@ export interface RequestSpendSeam { * on. Optional, so a seam built before it keeps compiling (BP-030). */ pending?(): RequestSpend; + /** + * Everything the request has spent across all its runs: every model call, + * and every harness run, including those an earlier run's suspension or + * interruption already handed on. Reads, never takes. Optional, so a seam + * built before it keeps compiling (BP-030). + */ + total?(): RequestSpend; } /** @@ -204,6 +211,23 @@ export function readRequestSpend(ctx: object): RequestSpend | undefined { return seam?.pending?.(); } +/** + * What the running request has spent across all its runs: every model call, + * and every harness run, however often it suspended, was interrupted or was + * continued. Unlike {@link readRequestSpend}, this includes what earlier + * endings already handed on, so a record of the whole request (a turn) reads + * the same after a resume as before it. + * + * Framework-facing, like {@link readRequestSpend}. + * + * @returns `undefined` when the context carries no spend ledger, or one built + * before it could read a total. + */ +export function readRequestTotalSpend(ctx: object): RequestSpend | undefined { + const seam = (ctx as { readonly [REQUEST_SPEND_SEAM]?: RequestSpendSeam })[REQUEST_SPEND_SEAM]; + return seam?.total?.(); +} + /** * One source of a request's spend (FIX-2145, shared by FIX-2288): a model, or * a harness billed one way. What {@link requestSpendSources} turns a diff --git a/packages/engine/src/context/createExecutionContext.ts b/packages/engine/src/context/createExecutionContext.ts index c872bd0638..8b1e193428 100644 --- a/packages/engine/src/context/createExecutionContext.ts +++ b/packages/engine/src/context/createExecutionContext.ts @@ -3167,8 +3167,8 @@ export async function createExecutionContext< // that run's to hand on, so it is carried rather than recovered. const loggedSpend = loggedSpendOf(requestRecord.items ?? [], requestRecord.spentThroughItemIndex ?? -1); const requestSpendLedger = interruptedEndingLive - ? createRequestSpendLedger(sumSpend(requestRecord.spentByModel ?? {}, loggedSpend.byModel)) - : createRequestSpendLedger(requestRecord.spentByModel, loggedSpend.byModel, loggedSpend.harnessRuns); + ? createRequestSpendLedger(sumSpend(requestRecord.spentByModel ?? {}, loggedSpend.byModel), {}, [], requestRecord.spentHarnessRuns) + : createRequestSpendLedger(requestRecord.spentByModel, loggedSpend.byModel, loggedSpend.harnessRuns, requestRecord.spentHarnessRuns); const requestEndRegistry = createRequestEndRegistry(() => requestSpendLedger.takeSinceLastEnding()); // How this request resumed (FIX-2129), read when a block asks: the // resolution is filled in at the checkpoint read, after this context is built. @@ -4204,7 +4204,8 @@ export async function createExecutionContext< if (at >= 0) harnessSpendOfBlock.splice(at, 1); harnessSpendOfBlock.push({ ...run }); }, - pending: () => requestSpendLedger.pending() + pending: () => requestSpendLedger.pending(), + total: () => requestSpendLedger.total() }; (childContext as { [REQUEST_RESUME_SEAM]?: () => RequestResume | undefined })[REQUEST_RESUME_SEAM] = readResume; // FIX-663: re-attach the background signal on every scope so nested diff --git a/packages/engine/src/execution/request-spend-ledger.ts b/packages/engine/src/execution/request-spend-ledger.ts index deb01d4508..0006519dd5 100644 --- a/packages/engine/src/execution/request-spend-ledger.ts +++ b/packages/engine/src/execution/request-spend-ledger.ts @@ -48,21 +48,26 @@ export interface RequestSpendLedger { * already handed on, so {@link RequestSpendLedger.takeSinceLastEnding} never * hands it on again. `recovered` and `recoveredHarnessRuns` are what a run that * never ended (its process died) left in the log: counted, and handed on at - * this run's first ending. + * this run's first ending. `carriedHarnessRuns` are the harness runs an + * earlier ending already handed on: in {@link RequestSpendLedger.total} like + * `carried`, never handed on again, and a later report under the same run id + * is handed on as its change since. */ export function createRequestSpendLedger( carried: Readonly> = {}, recovered: Readonly> = {}, - recoveredHarnessRuns: readonly HarnessRunSpend[] = [] + recoveredHarnessRuns: readonly HarnessRunSpend[] = [], + carriedHarnessRuns: readonly HarnessRunSpend[] = [] ): RequestSpendLedger { const byModel: Record = Object.assign(Object.create(null), copyOf(carried)); for (const [model, entry] of Object.entries(recovered)) addEntry((byModel[model] ??= emptyEntry()), entry); // One entry per run, as last reported, in the order last reported. A report // with no run id is a run of its own, under a key nothing else reports to; // so is each run the log recovered. - const harnessRuns = new Map(recoveredHarnessRuns.map((run, i) => [`recovered:${i}`, { ...run }])); + const carriedRuns = carriedHarnessRuns.map((run, i): [string, HarnessRunSpend] => [run.runId === undefined ? `carried:${i}` : `run:${run.runId}`, { ...run }]); + const harnessRuns = new Map([...carriedRuns, ...recoveredHarnessRuns.map((run, i): [string, HarnessRunSpend] => [`recovered:${i}`, { ...run }])]); // Each run as the last ending handed it on. - let takenRuns = new Map(); + let takenRuns = new Map(carriedRuns.map(([key, run]) => [key, { ...run }])); let takenByModel: Record = copyOf(carried); let settledRuns = 0; diff --git a/packages/engine/src/execution/runAction.ts b/packages/engine/src/execution/runAction.ts index 0f83e9038a..ffeb7a97f1 100644 --- a/packages/engine/src/execution/runAction.ts +++ b/packages/engine/src/execution/runAction.ts @@ -17,7 +17,7 @@ import { readDispatchStamp } from "./dispatch-metadata"; import { hasResumeMetadata, stripResumeMetadata } from "./resume-metadata"; import { RESUME_ACTION_STATUS } from "@flow-state-dev/core/types"; import { SuspensionError, errorDetailsWithCause, buildReplayLog, buildBlockInstanceId, blockPathObserver, parseBlockInstanceId, ROOT_BLOCK_PATH, resolveEntry as resolveTypedEntry, taskBindingOf, addressEntries } from "@flow-state-dev/core"; -import { SESSION_REQUEST_END_HOOK, onRequestEnded, type BackgroundFailure, type RequestEnding, type TaskEntry } from "@flow-state-dev/core/types"; +import { REQUEST_SPEND_SEAM, SESSION_REQUEST_END_HOOK, onRequestEnded, type BackgroundFailure, type HarnessRunSpend, type RequestEnding, type RequestSpendSeam, type TaskEntry } from "@flow-state-dev/core/types"; import type { ReplayLog } from "@flow-state-dev/core"; import type { BlockTraceItem, ContinuationItem, SuspensionItem, SuspensionResumeItem } from "@flow-state-dev/core/items"; import type { RuntimeItem } from "@flow-state-dev/core/items/internal"; @@ -605,6 +605,12 @@ async function patchRequestRecord( return writeRequestRecordPatch(stores, requestId, patch); } +/** Every harness run the request has spent so far, as its ledger holds them, for its record. */ +function spentHarnessRunsOf(ctx: object): HarnessRunSpend[] | undefined { + const runs = (ctx as { readonly [REQUEST_SPEND_SEAM]?: RequestSpendSeam })[REQUEST_SPEND_SEAM]?.total?.().harnessRuns; + return runs === undefined || runs.length === 0 ? undefined : runs.map((run) => ({ ...run })); +} + /** * Writes an interrupted request's settled spend once its ending has handed it * on (FIX-2145), only while the record is still `interrupted` under this @@ -614,7 +620,7 @@ async function recordSettledSpend( stores: StoreRegistry, requestId: string, incarnation: string, - spend: { spentByModel: RequestRecord["spentByModel"]; spentThroughItemIndex: number } + spend: { spentByModel: RequestRecord["spentByModel"]; spentHarnessRuns: RequestRecord["spentHarnessRuns"]; spentThroughItemIndex: number } ): Promise { const current = await stores.request.get(requestId); if (current?.status !== "interrupted" || (current.spentThroughItemIndex ?? -1) > spend.spentThroughItemIndex) return; @@ -2516,6 +2522,7 @@ async function runActionAttempt< await settleRequestRecord(options.stores, requestId, { status: "suspended" }, { items: suspendedItems, spentByModel: ctx.request.tokenUsage.byModel, + spentHarnessRuns: spentHarnessRunsOf(ctx), spentThroughItemIndex: lastItemIndexOf(suspendedItems) }); ctx.requestRuntime.status = "suspended"; @@ -3253,6 +3260,7 @@ async function runActionAttempt< if (ending.status === "interrupted" && terminalRecordIncarnation !== undefined) { await recordSettledSpend(options.stores, requestId, terminalRecordIncarnation, { spentByModel: ctx.request.tokenUsage.byModel, + spentHarnessRuns: spentHarnessRunsOf(ctx), spentThroughItemIndex: lastItemIndexOf(itemsToPersist()) }); } diff --git a/packages/engine/src/stores/types.ts b/packages/engine/src/stores/types.ts index bbbbc6687b..d7b4fd32a6 100644 --- a/packages/engine/src/stores/types.ts +++ b/packages/engine/src/stores/types.ts @@ -5,7 +5,7 @@ import type { SuspensionFilter, SuspensionRecord } from "@flow-state-dev/core/types"; -import type { JsonObject, ModelUsageEntry } from "@flow-state-dev/core/types"; +import type { HarnessRunSpend, JsonObject, ModelUsageEntry } from "@flow-state-dev/core/types"; import type { ExpectedVersion as SharedExpectedVersion } from "@flow-state-dev/core/helpers"; import type { BlockTraceItem, @@ -268,6 +268,14 @@ export type RequestRecord = ScopeRecordB * no settlement counted any, so every trace in the log counts. */ spentThroughItemIndex?: number; + /** + * The harness runs the request had spent when {@link RequestRecord.spentByModel} + * was written, as each was last reported. A resume or continuation starts + * its ledger's total from them, so a reader of the whole request's spend + * still counts a run an earlier ending handed on. Absent on a record written + * before it existed: no earlier harness run is carried (BP-030). + */ + spentHarnessRuns?: HarnessRunSpend[]; }; /** diff --git a/packages/engine/test/request-spend.test.ts b/packages/engine/test/request-spend.test.ts index 431b02a800..7fecd1bbf0 100644 --- a/packages/engine/test/request-spend.test.ts +++ b/packages/engine/test/request-spend.test.ts @@ -16,6 +16,7 @@ import { SESSION_REQUEST_END_HOOK, onRequestEnded, readRequestSpend, + readRequestTotalSpend, reportHarnessSpend, type FlowInstance, type GeneratorModel, @@ -986,7 +987,20 @@ describe("request spend read mid-run", () => { expect(ledger.total().byModel["model-a"]!.total).toBe(4); }); + // A resumed request's record carries the runs an earlier ending handed on: + // the whole request's spend still counts them, and no ending hands them on again. + it("counts a carried harness run in the total, never hands it on again, and hands on only a later report's change", () => { + const carried = { runId: "a", harness: "codex", inputTokens: 10, outputTokens: 2, costUsd: 0.1, billing: "api-key" as const }; + const ledger = createRequestSpendLedger({}, {}, [], [carried]); + expect(ledger.total().harnessRuns).toEqual([carried]); + expect(ledger.pending().harnessRuns).toEqual([]); + ledger.reportHarnessRun({ ...carried, inputTokens: 15, outputTokens: 3, costUsd: 0.15 }); + expect(ledger.pending().harnessRuns.map((run) => [run.runId, run.inputTokens, run.outputTokens])).toEqual([["a", 5, 1]]); + expect(ledger.total().harnessRuns.map((run) => [run.runId, run.inputTokens])).toEqual([["a", 15]]); + }); + it("reads undefined outside a request", () => { expect(readRequestSpend({})).toBeUndefined(); + expect(readRequestTotalSpend({})).toBeUndefined(); }); }); diff --git a/packages/harness-manager/src/session-flow.ts b/packages/harness-manager/src/session-flow.ts index 33441122f0..fb22331278 100644 --- a/packages/harness-manager/src/session-flow.ts +++ b/packages/harness-manager/src/session-flow.ts @@ -77,7 +77,7 @@ import type { HarnessUsageHook, HarnessUsageReader, } from "@flow-state-dev/core/types"; -import { readRequestSpend, type HarnessRunSpend, type ModelUsageEntry } from "@flow-state-dev/core/types"; +import { readRequestTotalSpend } from "@flow-state-dev/core/types"; import { recordBranchPullRequests, run } from "@flow-state-dev/workspace"; import { readRunChanges, runChangesSchema } from "./commits"; import { z } from "zod"; @@ -124,25 +124,6 @@ export const harnessTurnRecordSchema = z.object({ spend: taskSpendViewSchema.nullable().default(null), /** When the turn ended, in epoch millis: its duration is `endedAt - at`. */ endedAt: z.number().nullable().default(null), - /** - * The harness runs `spend` counts, as each last reported. Kept so a turn - * continued in a later run of its request (after an interruption) counts - * the runs before it too: a continued request's ledger holds only its own. - */ - spentRuns: z - .array( - z.object({ - runId: z.string().optional(), - harness: z.string(), - model: z.string().nullable().optional(), - inputTokens: z.number().nullable(), - outputTokens: z.number().nullable(), - costUsd: z.number().nullable(), - billing: z.enum(["api-key", "subscription"]).nullable(), - }), - ) - .nullable() - .default(null), }); /** One turn's record. */ @@ -297,7 +278,6 @@ const BLANK_TURN: HarnessTurnRecord = { checks: null, spend: null, endedAt: null, - spentRuns: null, }; /** `state` with the turn `requestId` changed by `change`, keeping only the latest {@link KEPT_TURNS}. */ @@ -460,7 +440,7 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { failure: harnessFailureKind(error), stopped: ctx.signal.aborted, error: error.message, - ...turnEndOf(ctx, requestId, record), + ...turnEndOf(ctx, requestId), })), ); // A stopped or failed turn still records where its changes end. Never in @@ -560,7 +540,7 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { execute: async (_handle: HarnessRunHandle, ctx): Promise => { const requestId = ctx.request.identity.id; try { - await changeState(ctx, (state) => withTurn(state, requestId, (record) => ({ ...record, ...turnEndOf(ctx, requestId, record) }))); + await changeState(ctx, (state) => withTurn(state, requestId, (record) => ({ ...record, ...turnEndOf(ctx, requestId) }))); } catch (error) { ctx.emit.status(`What the turn spent was not recorded: ${(error as Error).message}`, { transient: false }); } @@ -653,31 +633,13 @@ export function harnessSessionFlow(options: HarnessSessionFlowOptions) { /** * What a turn's record gets when its harness step ends: what it spent, and - * when. The runs are this run of the request's, as its ledger reports them, - * plus any an earlier run of the same request recorded that this one's - * ledger doesn't hold (a turn continued after an interruption: its ledger - * holds only its own runs). A run with an id is one run however often it is - * written, so a second write in the same run of the request counts it once. Model - * calls are the request's whole count (`ctx.request.tokenUsage`), not its - * pending slice: an interruption's ending hands its calls on, and a - * continuation replays them without making them again. + * when. The spend is the request's whole count, not what is left to hand on: + * a turn suspended, interrupted or continued spent what its earlier runs did + * too, and a replayed write reads the same number again. */ -function turnEndOf( - ctx: { request: { tokenUsage: { byModel: Record } } }, - requestId: string, - record: { spentRuns?: readonly HarnessRunSpend[] | null }, -): { spend: TaskSpendView | null; spentRuns: HarnessRunSpend[] | null; endedAt: number } { - const pending = readRequestSpend(ctx); - const now = pending?.harnessRuns ?? []; - // A run with an id is the same run wherever it is written, including one a - // continuation recovered from the log after this turn's earlier write (the - // engine names a run reported without an id, so that one has one too). One - // still without an id came from a seam that named none: a settled run of its - // own (the `HarnessRunSpend` contract), so an earlier one is always kept. - const earlier = (record.spentRuns ?? []).filter((run) => run.runId === undefined || !now.some((other) => other.runId === run.runId)); - const harnessRuns = [...earlier, ...now]; - const byModel = ctx.request.tokenUsage.byModel; - const spend = pending === undefined && harnessRuns.length === 0 ? null : requestSpendView({ byModel, harnessRuns }, requestId); - return { spend, spentRuns: harnessRuns.length === 0 ? null : harnessRuns, endedAt: Date.now() }; +function turnEndOf(ctx: object, requestId: string): { spend: TaskSpendView | null; endedAt: number } { + const total = readRequestTotalSpend(ctx); + const spend = total === undefined ? null : requestSpendView(total, requestId); + return { spend, endedAt: Date.now() }; } diff --git a/packages/harness-manager/test/turn-spend.spec.ts b/packages/harness-manager/test/turn-spend.spec.ts index 36257f0961..3938fcea3e 100644 --- a/packages/harness-manager/test/turn-spend.spec.ts +++ b/packages/harness-manager/test/turn-spend.spec.ts @@ -13,10 +13,9 @@ * - drop `onUsage` from the feeds: the usage query lists no window. * - write the spend from the flow's `onFinished` instead: the request's * terminal status is published before the spend lands. - * - keep a record's first end (skip a record that already ended), or count - * only the ledger's runs: a continued turn reads 10 tokens, not 20. - * - read model calls from the request's pending slice: a turn continued - * after a disconnect loses the calls its first run made. + * - read the request's pending slice instead of its total: a turn resumed + * after a suspension, or continued after a disconnect, loses what its + * first run spent. */ import { DEFAULT_ORG_ID, generator, handler, sequencer } from "@flow-state-dev/core"; import { reportHarnessSpend, type HarnessBlock, type HarnessCallbackContext } from "@flow-state-dev/core/types"; @@ -36,10 +35,6 @@ interface Script { fail?: boolean; /** A window heard during the run. */ window?: boolean; - /** An earlier run of this request already ended the turn with this run, as a turn continued after an interruption finds it. */ - earlier?: { runId?: string; inputTokens: number; outputTokens: number; costUsd: number | null }; - /** The run reports with no id, as a harness that settles each report does. */ - anonymous?: boolean; /** The run writes its report twice, as a harness reports its running total. */ reportTwice?: boolean; } @@ -51,16 +46,6 @@ const stubAgent = (script: Script) => (feeds: HarnessSessionFeeds) => execute: async (_input, ctx) => { const harnessCtx = ctx as unknown as HarnessCallbackContext; await feeds.resume(harnessCtx); - const earlier = script.earlier; - if (earlier !== undefined) { - const requestId = harnessCtx.request.identity.id; - await harnessCtx.session.atomicState((state) => { - const s = state as HarnessSessionState; - const turn = s.turns?.[requestId]; - const run = { harness: "stub", model: "stub-model", ...earlier, billing: "api-key" as const }; - return { ...s, turns: { ...s.turns, [requestId]: { ...turn!, endedAt: 1, spentRuns: [run] } } }; - }); - } if (script.window === true) { await feeds.onUsage( { harness: "stub", window: { id: "five_hour", label: "5h", minutes: 300, usedPercent: 42, level: "ok", resetsAt: null }, observedAt: Date.now(), runStartedAt: Date.now() - 10, accountId: "acct" }, @@ -73,7 +58,7 @@ const stubAgent = (script: Script) => (feeds: HarnessSessionFeeds) => } finally { if (script.spend !== undefined) { if (script.reportTwice === true) reportHarnessSpend(ctx, { runId: "run-1", harness: "stub", model: "stub-model", ...script.spend, inputTokens: 1, outputTokens: 1 }); - reportHarnessSpend(ctx, { ...(script.anonymous === true ? {} : { runId: "run-1" }), harness: "stub", model: "stub-model", ...script.spend }); + reportHarnessSpend(ctx, { runId: "run-1", harness: "stub", model: "stub-model", ...script.spend }); } } }, @@ -140,30 +125,66 @@ describe("a harness session turn's spend", () => { expect(turn.spend).toBeNull(); expect(turn.endedAt).not.toBeNull(); }); - // A turn continued after an interruption runs again under the same request, - // whose ledger then holds only the new run: the turn still counts both. - it("counts the runs of an earlier run of its request too, once each", async () => { + it("counts a run once however often its harness reports it", async () => { const stores = createInMemoryStores(); - await run(stores, "s-continued", { - earlier: { runId: "run-0", inputTokens: 10, outputTokens: 1, costUsd: 0.1 }, - reportTwice: true, - spend: { inputTokens: 10, outputTokens: 1, costUsd: 0.1, billing: "api-key" }, - }); - const turn = await onlyTurn(stores, "s-continued"); - expect(turn.spend).toMatchObject({ inputTokens: 20, outputTokens: 2 }); - expect(turn.spend?.costUsd).toBeCloseTo(0.2); - expect(turn.endedAt).toBeGreaterThan(1); + await run(stores, "s-twice", { reportTwice: true, spend: { inputTokens: 10, outputTokens: 1, costUsd: 0.1, billing: "api-key" } }); + const turn = await onlyTurn(stores, "s-twice"); + expect(turn.spend).toMatchObject({ inputTokens: 10, outputTokens: 1, costUsd: 0.1 }); }); - it("keeps an earlier run with no id even when its totals match this one's", async () => { + // A suspension (an approval the harness waits on) ends the request's first + // run and hands its spend on; the resumed run never runs the harness's + // earlier work again, so the turn's spend is the request's whole count. + it("keeps a run the harness reported before it suspended, once the turn resumes", async () => { const stores = createInMemoryStores(); - await run(stores, "s-anonymous", { - earlier: { inputTokens: 10, outputTokens: 1, costUsd: 0.1 }, - anonymous: true, - spend: { inputTokens: 10, outputTokens: 1, costUsd: 0.1, billing: "api-key" }, - }); - const turn = await onlyTurn(stores, "s-anonymous"); - expect(turn.spend).toMatchObject({ inputTokens: 20, outputTokens: 2 }); + await born(stores, "s-suspended"); + let runs = 0; + const agent = () => + sequencer({ name: "stub-harness", durable: true, inputSchema: z.object({ prompt: z.string() }) }) + .step( + handler({ + name: "work", + inputSchema: z.any(), + execute: (value, ctx) => { + runs += 1; + reportHarnessSpend(ctx, { runId: "run-0", harness: "stub", model: "stub-model", inputTokens: 10, outputTokens: 2, costUsd: 0.1, billing: "api-key" }); + return value; + }, + }), + ) + .step( + handler({ + name: "approve", + inputSchema: z.any(), + execute: async (_value, ctx) => { + await ctx.suspend!({ reason: "approval", message: "Go?" }); + return { outcome: "completed" }; + }, + }), + ) as unknown as HarnessBlock; + const flow = harnessSessionFlow({ kind: "stub-session", workspace, agent }); + const provider = createCheckpointDurabilityProvider({ checkpoints: stores.checkpoints, suspensions: stores.suspensions, leases: stores.leases }); + const runtimeConfig = { durabilityProvider: provider }; + + await runAction({ flow, actionName: "turn", input: { prompt: "hello" }, userId: "u", orgId: DEFAULT_ORG_ID, sessionId: "s-suspended", requestId: "r-suspended", stores, runtimeConfig } as never); + expect((await stores.request.get("r-suspended"))?.status).toBe("suspended"); + const [suspension] = (await provider.listSuspended({ status: "pending" })) as Array<{ suspensionId: string }>; + await provider.suspend({ ...(suspension as never as object), status: "approved", resolvedAt: Date.now(), resumeData: {} } as never); + const registry = createFlowRegistry(); + registry.register(flow as never); + const { finished } = await continueRequest({ + requestId: "r-suspended", + stores, + flowRegistry: registry, + resumeContext: { suspensionId: suspension!.suspensionId, action: "approve", data: {}, resumedBy: "u" }, + runtimeConfig, + } as never); + await finished; + + expect((await stores.request.get("r-suspended"))?.status).toBe("completed"); + expect(runs).toBe(1); + const turn = await onlyTurn(stores, "s-suspended"); + expect(turn.spend).toMatchObject({ harness: "stub", inputTokens: 10, outputTokens: 2, costUsd: 0.1 }); }); // A disconnect ends the request's first run, which hands its spend on; the From 4675799a1fa58fbca7363d815d176f45173b3f84 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 02:06:43 +0000 Subject: [PATCH 23/24] FIX-2398: a recovered report replaces its carried run; a live interrupted ending's logged runs are carried Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- .../src/context/createExecutionContext.ts | 2 +- .../src/execution/request-spend-ledger.ts | 33 ++++++-- packages/engine/test/request-spend.test.ts | 84 +++++++++++++++++++ 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/packages/engine/src/context/createExecutionContext.ts b/packages/engine/src/context/createExecutionContext.ts index 8b1e193428..15b1dd1028 100644 --- a/packages/engine/src/context/createExecutionContext.ts +++ b/packages/engine/src/context/createExecutionContext.ts @@ -3167,7 +3167,7 @@ export async function createExecutionContext< // that run's to hand on, so it is carried rather than recovered. const loggedSpend = loggedSpendOf(requestRecord.items ?? [], requestRecord.spentThroughItemIndex ?? -1); const requestSpendLedger = interruptedEndingLive - ? createRequestSpendLedger(sumSpend(requestRecord.spentByModel ?? {}, loggedSpend.byModel), {}, [], requestRecord.spentHarnessRuns) + ? createRequestSpendLedger(sumSpend(requestRecord.spentByModel ?? {}, loggedSpend.byModel), {}, [], [...(requestRecord.spentHarnessRuns ?? []), ...loggedSpend.harnessRuns]) : createRequestSpendLedger(requestRecord.spentByModel, loggedSpend.byModel, loggedSpend.harnessRuns, requestRecord.spentHarnessRuns); const requestEndRegistry = createRequestEndRegistry(() => requestSpendLedger.takeSinceLastEnding()); // How this request resumed (FIX-2129), read when a block asks: the diff --git a/packages/engine/src/execution/request-spend-ledger.ts b/packages/engine/src/execution/request-spend-ledger.ts index 0006519dd5..77ff47e155 100644 --- a/packages/engine/src/execution/request-spend-ledger.ts +++ b/packages/engine/src/execution/request-spend-ledger.ts @@ -50,8 +50,8 @@ export interface RequestSpendLedger { * never ended (its process died) left in the log: counted, and handed on at * this run's first ending. `carriedHarnessRuns` are the harness runs an * earlier ending already handed on: in {@link RequestSpendLedger.total} like - * `carried`, never handed on again, and a later report under the same run id - * is handed on as its change since. + * `carried`, never handed on again, and a later report under the same run id, + * reported here or recovered from the log, is handed on as its change since. */ export function createRequestSpendLedger( carried: Readonly> = {}, @@ -62,12 +62,17 @@ export function createRequestSpendLedger( const byModel: Record = Object.assign(Object.create(null), copyOf(carried)); for (const [model, entry] of Object.entries(recovered)) addEntry((byModel[model] ??= emptyEntry()), entry); // One entry per run, as last reported, in the order last reported. A report - // with no run id is a run of its own, under a key nothing else reports to; - // so is each run the log recovered. - const carriedRuns = carriedHarnessRuns.map((run, i): [string, HarnessRunSpend] => [run.runId === undefined ? `carried:${i}` : `run:${run.runId}`, { ...run }]); - const harnessRuns = new Map([...carriedRuns, ...recoveredHarnessRuns.map((run, i): [string, HarnessRunSpend] => [`recovered:${i}`, { ...run }])]); + // with no run id is a run of its own, under a key nothing else reports to. + // A run the log recovered under a carried run's id is that run's later + // report: it replaces the carried one, which stays what was handed on. + const carriedRuns = keyedRuns(carriedHarnessRuns, "carried"); + const harnessRuns = new Map(carriedRuns); + for (const [key, run] of keyedRuns(recoveredHarnessRuns, "recovered")) { + harnessRuns.delete(key); + harnessRuns.set(key, run); + } // Each run as the last ending handed it on. - let takenRuns = new Map(carriedRuns.map(([key, run]) => [key, { ...run }])); + let takenRuns = new Map([...carriedRuns].map(([key, run]) => [key, { ...run }])); let takenByModel: Record = copyOf(carried); let settledRuns = 0; @@ -123,6 +128,20 @@ export function createRequestSpendLedger( }; } +/** + * `runs` by ledger key, each run at its last report: `run:`, or + * `:` for a run that names no id. + */ +function keyedRuns(runs: readonly HarnessRunSpend[], prefix: string): Map { + const keyed = new Map(); + runs.forEach((run, i) => { + const key = run.runId === undefined ? `${prefix}:${i}` : `run:${run.runId}`; + keyed.delete(key); + keyed.set(key, { ...run }); + }); + return keyed; +} + /** Whether two reports of one run say the same. */ function sameRun(a: HarnessRunSpend, b: HarnessRunSpend): boolean { return ( diff --git a/packages/engine/test/request-spend.test.ts b/packages/engine/test/request-spend.test.ts index 7fecd1bbf0..564f456e48 100644 --- a/packages/engine/test/request-spend.test.ts +++ b/packages/engine/test/request-spend.test.ts @@ -609,6 +609,81 @@ describe("request spend ledger", () => { expect(total).toBe(calls * 3); }); + it("keeps a harness run in a continuation's total when the interrupted run is still ending, and hands it on once", async () => { + // The continuation replays the finished harness step without running it, + // and the interrupted run's record holds no runs until its ending is done: + // the log is where the continuation's total reads the run from. + process.env.FSDEV_TRACE_OBSERVABILITY = "false"; + const endings: RequestEnding[] = []; + let totalAtEnd: unknown; + let waited = false; + const harnessStep = handler({ + name: "harness-step", + inputSchema: z.any(), + execute: (value, ctx) => { + reportHarnessSpend(ctx, { runId: "run-1", harness: "codex", inputTokens: 10, outputTokens: 2, costUsd: 0.1, billing: "api-key" }); + return value; + } + }); + const waitOnce = handler({ + name: "wait-once", + inputSchema: z.any(), + execute: async (value, ctx) => { + if (waited) return value; + waited = true; + await new Promise((_resolve, reject) => ctx.signal.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError")))); + return value; + } + }); + const readTotal = handler({ + name: "read-total", + inputSchema: z.any(), + execute: (value, ctx) => { + totalAtEnd = readRequestTotalSpend(ctx)?.harnessRuns.map((run) => [run.runId, run.inputTokens, run.costUsd]); + return value; + } + }); + let releaseObserver: () => void = () => {}; + let observerHeld = false; + const holdFirst = handler({ + name: "hold-first", + inputSchema: z.any(), + execute: async () => { + if (observerHeld) return; + observerHeld = true; + await new Promise((resolve) => (releaseObserver = resolve)); + } + }); + const flow = defineFlow({ + kind: "spend-live-harness", + session: { [SESSION_REQUEST_END_HOOK]: () => (ending) => void endings.push(ending) }, + request: { onFinished: holdFirst }, + actions: { + run: { inputSchema: z.object({}), block: sequencer({ name: "seq", durable: true }).step(harnessStep).step(waitOnce).step(readTotal) } + } + })({ id: "spend-live-harness" }); + const stores = createInMemoryStores(); + const provider = createCheckpointDurabilityProvider({ checkpoints: stores.checkpoints, suspensions: stores.suspensions, leases: stores.leases }); + const runtimeConfig = { durabilityProvider: provider }; + const disconnect = new AbortController(); + + const older = runAction({ orgId: DEFAULT_ORG_ID, flow, actionName: "run", input: {}, userId: "u1", requestId: "r-live-h", signal: disconnect.signal, stores, runtimeConfig }); + while (!waited) await new Promise((resolve) => setTimeout(resolve, 5)); + disconnect.abort(); + while (!observerHeld) await new Promise((resolve) => setTimeout(resolve, 5)); + await stores.request.flushItems("r-live-h"); + const registry = createFlowRegistry(); + registry.register(flow as never); + const { finished } = await continueRequest({ requestId: "r-live-h", stores, flowRegistry: registry, runtimeConfig }); + await finished; + releaseObserver(); + await older; + + expect(totalAtEnd).toEqual([["run-1", 10, 0.1]]); + // The interrupted run hands the run on; the continuation does not again. + expect(endings.flatMap((ending) => ending.spend.harnessRuns.map((run) => run.inputTokens))).toEqual([10]); + }); + it("hands on an interrupted run's spend at the continuation when that run's process died before it ended", async () => { // The interrupted run settled its record, spent 3 in its observer, then its // process died before its ending ran: nothing it spent was handed on. A @@ -999,6 +1074,15 @@ describe("request spend read mid-run", () => { expect(ledger.total().harnessRuns.map((run) => [run.runId, run.inputTokens])).toEqual([["a", 15]]); }); + // A suspended request resumes a run, the run reports again, and the process + // dies: the log's newer report is that run's, not a second run. + it("takes a recovered report of a carried run as that run's latest, and hands on only its change", () => { + const carried = { runId: "a", harness: "codex", inputTokens: 10, outputTokens: 2, costUsd: 0.1, billing: "api-key" as const }; + const ledger = createRequestSpendLedger({}, {}, [{ ...carried, inputTokens: 20, outputTokens: 4, costUsd: 0.2 }], [carried]); + expect(ledger.total().harnessRuns.map((run) => [run.runId, run.inputTokens, run.costUsd])).toEqual([["a", 20, 0.2]]); + expect(ledger.takeSinceLastEnding().harnessRuns.map((run) => [run.runId, run.inputTokens, run.costUsd])).toEqual([["a", 10, 0.1]]); + }); + it("reads undefined outside a request", () => { expect(readRequestSpend({})).toBeUndefined(); expect(readRequestTotalSpend({})).toBeUndefined(); From 340c9d380a598b5226c5449a1ae145f373893f52 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 02:12:54 +0000 Subject: [PATCH 24/24] FIX-2398: a continued request keeps a dead run's later charge when only its charge changed Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NczDiDfgRv9riWiVVdj8JG --- packages/orchestration/src/tasks/spend.ts | 4 +++- .../orchestration/test/collection/record-spend.test.ts | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/orchestration/src/tasks/spend.ts b/packages/orchestration/src/tasks/spend.ts index b9e7563e9a..a5d27bca2c 100644 --- a/packages/orchestration/src/tasks/spend.ts +++ b/packages/orchestration/src/tasks/spend.ts @@ -171,6 +171,8 @@ function unrecovered(live: TaskSpendSource[], recovered: TaskSpendSource[] | nul // The continuation reports the logged charge itself, as it does the tokens. costUsd: source.costUsd === null ? null : left(source.costUsd, logged.costUsd ?? 0), }; - return kept.inputTokens === 0 && kept.outputTokens === 0 && (source.kind === "model" || kept.runs === 0) ? [] : [kept]; + // A charge left with no tokens is a run's later report the log missed: still owed. + const charged = kept.costUsd !== null && kept.costUsd > 0; + return !charged && kept.inputTokens === 0 && kept.outputTokens === 0 && (source.kind === "model" || kept.runs === 0) ? [] : [kept]; }); } diff --git a/packages/orchestration/test/collection/record-spend.test.ts b/packages/orchestration/test/collection/record-spend.test.ts index 4232ff49eb..f4e314a5a7 100644 --- a/packages/orchestration/test/collection/record-spend.test.ts +++ b/packages/orchestration/test/collection/record-spend.test.ts @@ -188,6 +188,16 @@ describe.each([ expect(view(collection)?.costUsd).toBeCloseTo(0.2); }); + it("keeps a dead run's later charge when only its charge changed after the log's report", async () => { + // The run's live report raised its charge to $0.20 with the same tokens; the log holds the $0.10 report. + const { collection, claim } = await claimed(); + const run = (costUsd: number): RequestSpend => ({ byModel: {}, harnessRuns: [{ runId: "r", harness: "claude-code", model: "claude-sonnet-4-5", inputTokens: 10, outputTokens: 2, costUsd, billing: "api-key" }] }); + await collection.recordSpend!("t", { requestId: "q1", part: "live", spend: run(0.2), writer: "first", admittedAt: 1 }, { claim }); + await collection.recordSpend!("t", { requestId: "q1", part: "live", spend: run(0.1), writer: "second", admittedAt: 2, recovered: run(0.1) }, { claim }); + expect(view(collection)).toMatchObject({ inputTokens: 10, outputTokens: 2 }); + expect(view(collection)?.costUsd).toBeCloseTo(0.2); + }); + it("keeps a dead run's harness tokens when its request is continued in a new process", async () => { // A harness run whose block never finished isn't in the log, so the continuation doesn't recover it: the row is its only record. const { collection, claim } = await claimed();